From c9f33c9c181c714fbf2f7924acb58312fead996e Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Tue, 17 Jan 2023 14:25:42 +0100 Subject: [PATCH 01/21] YARN-11041. Replace all occurences of queuePath with the new QueuePath class - followup Change-Id: I2d080bb414b08c2cae6a83d8d043667464451fc9 --- .../yarn/client/cli/TestSchedConfCLI.java | 3 +- .../AbstractAutoCreatedLeafQueue.java | 6 +- .../scheduler/capacity/AbstractCSQueue.java | 2 +- .../scheduler/capacity/AbstractLeafQueue.java | 8 +- .../capacity/AutoCreatedQueueTemplate.java | 25 ++-- .../scheduler/capacity/CSQueueUtils.java | 2 +- .../CapacitySchedulerConfigValidator.java | 2 +- .../CapacitySchedulerConfiguration.java | 125 +++++++----------- .../CapacitySchedulerQueueManager.java | 2 +- .../capacity/ManagedParentQueue.java | 15 +-- .../scheduler/capacity/PlanQueue.java | 3 +- .../scheduler/capacity/QueuePrefixes.java | 41 ++++++ .../scheduler/capacity/UserWeights.java | 2 +- .../conf/QueueCapacityConfigParser.java | 3 +- .../dao/CapacitySchedulerQueueInfo.java | 3 +- .../webapp/dao/LeafQueueTemplateInfo.java | 3 +- .../resourcemanager/TestAppManager.java | 46 ++++--- .../TestApplicationMasterServiceCapacity.java | 10 +- ...pacitySchedulerConfigGeneratorForTest.java | 16 +-- .../CapacitySchedulerQueueHelpers.java | 10 ++ .../capacity/TestApplicationLimits.java | 9 +- .../TestApplicationLimitsByPartition.java | 12 +- .../TestAutoCreatedQueueTemplate.java | 102 +++++++------- .../TestCSAllocateCustomResource.java | 2 +- .../capacity/TestCapacityScheduler.java | 19 +-- .../capacity/TestCapacitySchedulerApps.java | 2 +- ...CapacitySchedulerAutoCreatedQueueBase.java | 2 +- .../TestCapacitySchedulerConfiguration.java | 3 +- .../TestCapacitySchedulerMultiNodes.java | 7 +- ...CapacitySchedulerNewQueueAutoCreation.java | 66 +++++---- .../TestCapacitySchedulerOvercommit.java | 9 +- .../capacity/TestCapacitySchedulerPerf.java | 13 +- .../capacity/TestCapacitySchedulerQueues.java | 82 +++++------- .../TestCapacitySchedulerWeightMode.java | 72 +++++----- ...pacitySchedulerWithMultiResourceTypes.java | 11 +- .../capacity/TestContainerAllocation.java | 2 +- .../scheduler/capacity/TestQueueParsing.java | 23 ++-- .../scheduler/capacity/TestUtils.java | 3 +- .../conf/TestQueueCapacityConfigParser.java | 34 +++-- .../policy/TestFairOrderingPolicy.java | 8 +- .../TestRMWebServicesCapacitySched.java | 3 +- ...WebServicesCapacitySchedDynamicConfig.java | 3 +- ...estRMWebServicesConfigurationMutation.java | 12 +- ...edulerActivitiesWithMultiNodesEnabled.java | 11 +- 44 files changed, 434 insertions(+), 403 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java index 4df20da68ccca..369f22c97641b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.client.cli; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.junit.Before; import org.junit.Test; @@ -182,7 +183,7 @@ private static void setupQueueConfiguration( CapacitySchedulerConfiguration config) { config.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"testqueue"}); - String a = CapacitySchedulerConfiguration.ROOT + ".testqueue"; + QueuePath a = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".testqueue"); config.setCapacity(a, 100f); config.setMaximumCapacity(a, 100f); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index 15960c8de8494..5d3aaefe50c0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -63,7 +63,7 @@ public void setEntitlement(QueueEntitlement entitlement) @Override protected Resource getMinimumAbsoluteResource(String queuePath, String label) { - return super.getMinimumAbsoluteResource(queueContext.getConfiguration() + return super.getMinimumAbsoluteResource(QueuePrefixes .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), label); } @@ -71,7 +71,7 @@ protected Resource getMinimumAbsoluteResource(String queuePath, @Override protected Resource getMaximumAbsoluteResource(String queuePath, String label) { - return super.getMaximumAbsoluteResource(queueContext.getConfiguration() + return super.getMaximumAbsoluteResource(QueuePrefixes .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), label); } @@ -79,7 +79,7 @@ protected Resource getMaximumAbsoluteResource(String queuePath, @Override protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, String label) { - return super.checkConfigTypeIsAbsoluteResource(queueContext.getConfiguration() + return super.checkConfigTypeIsAbsoluteResource(QueuePrefixes .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), label); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 7a3ec4f6f4e5e..f1eabab56aa5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -408,7 +408,7 @@ protected void setDynamicQueueProperties() { // Set properties from parent template if (parent instanceof ParentQueue) { ((ParentQueue) parent).getAutoCreatedQueueTemplate() - .setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePath()); + .setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePathObject()); String parentTemplate = String.format("%s.%s", parent.getQueuePath(), AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java index 08fedb578cab9..fcf102e35ce54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java @@ -195,9 +195,9 @@ protected void setupQueueConfigs(Resource clusterResource) throws maxAMResourcePerQueuePercent = configuration.getMaximumApplicationMasterResourcePerQueuePercent( - getQueuePath()); + getQueuePathObject()); - maxApplications = configuration.getMaximumApplicationsPerQueue(getQueuePath()); + maxApplications = configuration.getMaximumApplicationsPerQueue(getQueuePathObject()); if (maxApplications < 0) { int maxGlobalPerQueueApps = configuration.getGlobalMaximumApplicationsPerQueue(); @@ -1697,7 +1697,7 @@ protected void setDynamicQueueProperties() { queueContext.getConfiguration().setUserLimitFactor(getQueuePath(), -1); // Set Max AM percentage to a higher value queueContext.getConfiguration().setMaximumApplicationMasterResourcePerQueuePercent( - getQueuePath(), 1f); + getQueuePathObject(), 1f); super.setDynamicQueueProperties(); } @@ -2343,7 +2343,7 @@ public void stopQueue() { void updateMaximumApplications() { CapacitySchedulerConfiguration configuration = queueContext.getConfiguration(); - int maxAppsForQueue = configuration.getMaximumApplicationsPerQueue(getQueuePath()); + int maxAppsForQueue = configuration.getMaximumApplicationsPerQueue(getQueuePathObject()); int maxDefaultPerQueueApps = configuration.getGlobalMaximumApplicationsPerQueue(); int maxSystemApps = configuration.getMaximumSystemApplications(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index bbeca4b072450..c8099190081cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -28,7 +28,7 @@ import org.apache.hadoop.util.Lists; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix; /** * A handler for storing and setting auto created queue template settings. @@ -53,9 +53,8 @@ public AutoCreatedQueueTemplate(CapacitySchedulerConfiguration configuration, } @VisibleForTesting - public static String getAutoQueueTemplatePrefix(String queue) { - return CapacitySchedulerConfiguration.getQueuePrefix(queue) - + AUTO_QUEUE_TEMPLATE_PREFIX; + public static String getAutoQueueTemplatePrefix(QueuePath queuePath) { + return getQueuePrefix(queuePath) + AUTO_QUEUE_TEMPLATE_PREFIX; } /** @@ -89,7 +88,7 @@ public Map getParentOnlyProperties() { * @param childQueuePath child queue path used for prefixing the properties */ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, - String childQueuePath) { + QueuePath childQueuePath) { setTemplateEntriesForChild(conf, childQueuePath, false); } @@ -103,7 +102,7 @@ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, * @param childQueuePath child queue path used for prefixing the properties */ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, - String childQueuePath, + QueuePath childQueuePath, boolean isLeaf) { if (childQueuePath.equals(ROOT)) { return; @@ -114,8 +113,7 @@ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, // Get all properties that are explicitly set Set alreadySetProps = configurationProperties - .getPropertiesWithPrefix(CapacitySchedulerConfiguration - .getQueuePrefix(childQueuePath)).keySet(); + .getPropertiesWithPrefix(getQueuePrefix(childQueuePath)).keySet(); // Check template properties only set for leaf or parent queues Map queueTypeSpecificTemplates = parentOnlyProperties; @@ -129,8 +127,7 @@ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, if (alreadySetProps.contains(entry.getKey())) { continue; } - conf.set(CapacitySchedulerConfiguration.getQueuePrefix( - childQueuePath) + entry.getKey(), entry.getValue()); + conf.set(getQueuePrefix(childQueuePath) + entry.getKey(), entry.getValue()); } for (Map.Entry entry : templateProperties.entrySet()) { @@ -140,8 +137,7 @@ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, || queueTypeSpecificTemplates.containsKey(entry.getKey())) { continue; } - conf.set(CapacitySchedulerConfiguration.getQueuePrefix( - childQueuePath) + entry.getKey(), entry.getValue()); + conf.set(getQueuePrefix(childQueuePath) + entry.getKey(), entry.getValue()); } } @@ -169,7 +165,7 @@ private void setTemplateConfigEntries(CapacitySchedulerConfiguration configurati int wildcardLevel = 0; // root can not be wildcarded int supportedWildcardLevel = Math.min(queuePathMaxIndex, - configuration.getMaximumAutoCreatedQueueDepth(queuePath.getFullPath())); + configuration.getMaximumAutoCreatedQueueDepth(queuePath)); // Allow root to have template properties if (queuePath.isRoot()) { supportedWildcardLevel = 0; @@ -177,7 +173,8 @@ private void setTemplateConfigEntries(CapacitySchedulerConfiguration configurati // Collect all template entries while (wildcardLevel <= supportedWildcardLevel) { - String templateQueuePath = String.join(".", queuePathParts); + String queuePathString = String.join(".", queuePathParts); + QueuePath templateQueuePath = new QueuePath(queuePathString); // Get all configuration entries with // yarn.scheduler.capacity. prefix Map queueProps = configurationProperties diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index ecb285748b510..df75b2309b19e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -80,7 +80,7 @@ public static void loadCapacitiesByLabelsFromConf( label, csConf.getMaximumAMResourcePercentPerPartition(queuePath, label)); queueCapacities.setWeight(label, - csConf.getNonLabeledQueueWeight(queuePath.getFullPath())); + csConf.getNonLabeledQueueWeight(queuePath)); } else{ queueCapacities.setCapacity(label, csConf.getLabeledQueueCapacity(queuePath, label) / 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index 5ddc709b57c9c..9748679354e88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -131,7 +131,7 @@ public static void validateQueueHierarchy( } final String queuePath = oldQueue.getQueuePath(); - final String configPrefix = CapacitySchedulerConfiguration.getQueuePrefix( + final String configPrefix = QueuePrefixes.getQueuePrefix( oldQueue.getQueuePath()); final QueueState newQueueState = createQueueState(newConf.get(configPrefix + "state"), queuePath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 51616da14b60b..a050825ca115a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping; @@ -78,6 +77,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.*; + public class CapacitySchedulerConfiguration extends ReservationSchedulerConfiguration { private static final Logger LOG = @@ -426,12 +427,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur private ConfigurationProperties configurationProperties; - public int getMaximumAutoCreatedQueueDepth(String queuePath) { + public int getMaximumAutoCreatedQueueDepth(QueuePath queuePath) { return getInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH, getInt(PREFIX + MAXIMUM_QUEUE_DEPTH, DEFAULT_MAXIMUM_QUEUE_DEPTH)); } - public void setMaximumAutoCreatedQueueDepth(String queuePath, int value) { + public void setMaximumAutoCreatedQueueDepth(QueuePath queuePath, int value) { setInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH, value); } @@ -464,27 +465,10 @@ public CapacitySchedulerConfiguration(Configuration configuration, } } - public static String getQueuePrefix(String queue) { - String queueName = PREFIX + queue + DOT; - return queueName; - } - - static String getQueueOrderingPolicyPrefix(String queue) { - String queueName = PREFIX + queue + DOT + ORDERING_POLICY + DOT; - return queueName; - } - static String getUserPrefix(String user) { return PREFIX + "user." + user + DOT; } - public static String getNodeLabelPrefix(String queue, String label) { - if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { - return getQueuePrefix(queue); - } - return getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT; - } - public void setMaximumSystemApplications(int numMaxApps) { setInt(MAXIMUM_SYSTEM_APPLICATIONS, numMaxApps); } @@ -503,73 +487,73 @@ public float getMaximumApplicationMasterResourcePercent() { /** * Get the maximum applications per queue setting. - * @param queue name of the queue + * @param queuePath path of the queue * @return setting specified or -1 if not set */ - public int getMaximumApplicationsPerQueue(String queue) { + public int getMaximumApplicationsPerQueue(QueuePath queuePath) { int maxApplicationsPerQueue = - getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX, + getInt(getQueuePrefix(queuePath) + MAXIMUM_APPLICATIONS_SUFFIX, (int)UNDEFINED); return maxApplicationsPerQueue; } @VisibleForTesting - public void setMaximumApplicationsPerQueue(String queue, + public void setMaximumApplicationsPerQueue(QueuePath queuePath, int numMaxApps) { - setInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX, + setInt(getQueuePrefix(queuePath) + MAXIMUM_APPLICATIONS_SUFFIX, numMaxApps); } /** * Get the maximum am resource percent per queue setting. - * @param queue name of the queue + * @param queuePath path of the queue * @return per queue setting or defaults to the global am-resource-percent * setting if per queue setting not present */ - public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) { - return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, + public float getMaximumApplicationMasterResourcePerQueuePercent(QueuePath queuePath) { + return getFloat(getQueuePrefix(queuePath) + MAXIMUM_AM_RESOURCE_SUFFIX, getMaximumApplicationMasterResourcePercent()); } - public void setMaximumApplicationMasterResourcePerQueuePercent(String queue, + public void setMaximumApplicationMasterResourcePerQueuePercent(QueuePath queuePath, float percent) { - setFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, percent); + setFloat(getQueuePrefix(queuePath) + MAXIMUM_AM_RESOURCE_SUFFIX, percent); } - private void throwExceptionForUnexpectedWeight(float weight, String queue, + private void throwExceptionForUnexpectedWeight(float weight, QueuePath queuePath, String label) { if ((weight < -1e-6 && Math.abs(weight + 1) > 1e-6) || weight > 10000) { throw new IllegalArgumentException( - "Illegal " + "weight=" + weight + " for queue=" + queue + "label=" + "Illegal " + "weight=" + weight + " for queue=" + queuePath.getFullPath() + "label=" + label + ". Acceptable values: [0, 10000], -1 is same as not set"); } } - public float getNonLabeledQueueWeight(String queue) { - String configuredValue = get(getQueuePrefix(queue) + CAPACITY); + public float getNonLabeledQueueWeight(QueuePath queuePath) { + String configuredValue = get(getQueuePrefix(queuePath) + CAPACITY); float weight = extractFloatValueFromWeightConfig(configuredValue); - throwExceptionForUnexpectedWeight(weight, queue, ""); + throwExceptionForUnexpectedWeight(weight, queuePath, ""); return weight; } - public void setNonLabeledQueueWeight(String queue, float weight) { - set(getQueuePrefix(queue) + CAPACITY, weight + WEIGHT_SUFFIX); + public void setNonLabeledQueueWeight(QueuePath queuePath, float weight) { + set(getQueuePrefix(queuePath) + CAPACITY, weight + WEIGHT_SUFFIX); } - public void setLabeledQueueWeight(String queue, String label, float weight) { - set(getNodeLabelPrefix(queue, label) + CAPACITY, weight + WEIGHT_SUFFIX); + public void setLabeledQueueWeight(QueuePath queuePath, String label, float weight) { + set(getNodeLabelPrefix(queuePath, label) + CAPACITY, weight + WEIGHT_SUFFIX); } - public float getLabeledQueueWeight(QueuePath queue, String label) { - String configuredValue = get(getNodeLabelPrefix(queue.getFullPath(), label) + CAPACITY); + public float getLabeledQueueWeight(QueuePath queuePath, String label) { + String configuredValue = get(getNodeLabelPrefix(queuePath, label) + CAPACITY); float weight = extractFloatValueFromWeightConfig(configuredValue); - throwExceptionForUnexpectedWeight(weight, queue.getFullPath(), label); + throwExceptionForUnexpectedWeight(weight, queuePath, label); return weight; } - public float getNonLabeledQueueCapacity(QueuePath queue) { - String configuredCapacity = get(getQueuePrefix(queue.getFullPath()) + CAPACITY); + public float getNonLabeledQueueCapacity(QueuePath queuePath) { + String configuredCapacity = get(getQueuePrefix(queuePath) + CAPACITY); boolean absoluteResourceConfigured = (configuredCapacity != null) && RESOURCE_PATTERN.matcher(configuredCapacity).find(); if (absoluteResourceConfigured || configuredWeightAsCapacity( @@ -578,10 +562,10 @@ public float getNonLabeledQueueCapacity(QueuePath queue) { // root.From AbstractCSQueue, absolute resource will be parsed and // updated. Once nodes are added/removed in cluster, capacity in // percentage will also be re-calculated. - return queue.isRoot() ? 100.0f : 0f; + return queuePath.isRoot() ? 100.0f : 0f; } - float capacity = queue.isRoot() + float capacity = queuePath.isRoot() ? 100.0f : (configuredCapacity == null) ? 0f @@ -589,39 +573,39 @@ public float getNonLabeledQueueCapacity(QueuePath queue) { if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException( - "Illegal " + "capacity of " + capacity + " for queue " + queue); + "Illegal " + "capacity of " + capacity + " for queue " + queuePath); } LOG.debug("CSConf - getCapacity: queuePrefix={}, capacity={}", - getQueuePrefix(queue.getFullPath()), capacity); + getQueuePrefix(queuePath), capacity); return capacity; } - public void setCapacity(String queue, float capacity) { - if (queue.equals("root")) { + public void setCapacity(QueuePath queuePath, float capacity) { + if (queuePath.getFullPath().equals("root")) { throw new IllegalArgumentException( "Cannot set capacity, root queue has a fixed capacity of 100.0f"); } - setFloat(getQueuePrefix(queue) + CAPACITY, capacity); + setFloat(getQueuePrefix(queuePath) + CAPACITY, capacity); LOG.debug("CSConf - setCapacity: queuePrefix={}, capacity={}", - getQueuePrefix(queue), capacity); + getQueuePrefix(queuePath), capacity); } @VisibleForTesting - public void setCapacity(String queue, String absoluteResourceCapacity) { - if (queue.equals("root")) { + public void setCapacity(QueuePath queuePath, String absoluteResourceCapacity) { + if (queuePath.getFullPath().equals("root")) { throw new IllegalArgumentException( "Cannot set capacity, root queue has a fixed capacity"); } - set(getQueuePrefix(queue) + CAPACITY, absoluteResourceCapacity); + set(getQueuePrefix(queuePath) + CAPACITY, absoluteResourceCapacity); LOG.debug("CSConf - setCapacity: queuePrefix={}, capacity={}", - getQueuePrefix(queue), absoluteResourceCapacity); + getQueuePrefix(queuePath), absoluteResourceCapacity); } - public float getNonLabeledQueueMaximumCapacity(QueuePath queue) { - String configuredCapacity = get(getQueuePrefix(queue.getFullPath()) + MAXIMUM_CAPACITY); + public float getNonLabeledQueueMaximumCapacity(QueuePath queuePath) { + String configuredCapacity = get(getQueuePrefix(queuePath) + MAXIMUM_CAPACITY); boolean matcher = (configuredCapacity != null) && RESOURCE_PATTERN.matcher(configuredCapacity).find(); if (matcher) { @@ -641,14 +625,14 @@ public float getNonLabeledQueueMaximumCapacity(QueuePath queue) { return maxCapacity; } - public void setMaximumCapacity(String queue, float maxCapacity) { + public void setMaximumCapacity(QueuePath queuePath, float maxCapacity) { if (maxCapacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException("Illegal " + - "maximum-capacity of " + maxCapacity + " for queue " + queue); + "maximum-capacity of " + maxCapacity + " for queue " + queuePath.getFullPath()); } - setFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); + setFloat(getQueuePrefix(queuePath) + MAXIMUM_CAPACITY, maxCapacity); LOG.debug("CSConf - setMaxCapacity: queuePrefix={}, maxCapacity={}", - getQueuePrefix(queue), maxCapacity); + getQueuePrefix(queuePath), maxCapacity); } public void setCapacityByLabel(String queue, String label, float capacity) { @@ -2293,23 +2277,6 @@ public boolean isAutoQueueCreationV2Enabled(String queuePath) { return isAutoQueueCreation; } - /** - * Get the auto created leaf queue's template configuration prefix - * Leaf queue's template capacities are configured at the parent queue - * - * @param queuePath parent queue's path - * @return Config prefix for leaf queue template configurations - */ - @Private - public String getAutoCreatedQueueTemplateConfPrefix(String queuePath) { - return queuePath + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX; - } - - @Private - public QueuePath getAutoCreatedQueueObjectTemplateConfPrefix(String queuePath) { - return new QueuePath(queuePath, AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX); - } - @Private public static final String FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY = "auto-create-child-queue.fail-on-exceeding-parent-capacity"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index ba6849cb7807b..e74ac47dc841c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -571,7 +571,7 @@ public List determineMissingParents( } int maximumDepthOfStaticParent = csContext.getConfiguration().getMaximumAutoCreatedQueueDepth( - firstExistingStaticParent.getQueuePath()); + firstExistingStaticParent.getQueuePathObject()); if (firstStaticParentDistance > maximumDepthOfStaticParent) { throw new SchedulerDynamicEditException( "Could not auto create queue " + queue.getFullPath() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 6867c6baf9d7e..32a33bff5a4e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -156,8 +156,7 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws queueContext.getConfiguration(); // TODO load configs into CapacitySchedulerConfiguration instead of duplicating them - String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix( - configuration); + String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(); //Load template configuration into CapacitySchedulerConfiguration CapacitySchedulerConfiguration autoCreatedTemplateConfig = super.initializeLeafQueueConfigs(leafQueueTemplateConfPrefix); @@ -165,7 +164,7 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws QueueResourceQuotas queueResourceQuotas = new QueueResourceQuotas(); setAbsoluteResourceTemplates(configuration, queueResourceQuotas); - QueuePath templateQueuePath = configuration + QueuePath templateQueuePath = QueuePrefixes .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath()); Set templateConfiguredNodeLabels = queueContext .getQueueManager().getConfiguredNodeLabelsForAllQueues() @@ -193,7 +192,7 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws private void setAbsoluteResourceTemplates(CapacitySchedulerConfiguration configuration, QueueResourceQuotas queueResourceQuotas) throws IOException { - QueuePath templateQueuePath = configuration + QueuePath templateQueuePath = QueuePrefixes .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath()); Set templateConfiguredNodeLabels = queueContext .getQueueManager().getConfiguredNodeLabelsForAllQueues() @@ -222,14 +221,14 @@ private void updateQueueCapacities(QueueCapacities queueCapacities) { queueContext.getClusterResource(), configuration.getMinimumResourceRequirement( label, - configuration + QueuePrefixes .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), resourceTypes), getQueueResourceQuotas().getConfiguredMinResource(label))); Resource childMaxResource = configuration .getMaximumResourceRequirement(label, - configuration + QueuePrefixes .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), resourceTypes); Resource parentMaxRes = getQueueResourceQuotas() @@ -379,8 +378,8 @@ public List getAllApplications() { } } - public String getLeafQueueConfigPrefix(CapacitySchedulerConfiguration conf) { - return CapacitySchedulerConfiguration.PREFIX + conf + public String getLeafQueueConfigPrefix() { + return CapacitySchedulerConfiguration.PREFIX + QueuePrefixes .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index 3bbbffe588192..eab94436d7f8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -52,7 +52,8 @@ public PlanQueue(CapacitySchedulerQueueContext queueContext, String queueName, // Set the reservation queue attributes for the Plan CapacitySchedulerConfiguration conf = queueContext.getConfiguration(); String queuePath = super.getQueuePath(); - int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePath); + QueuePath queuePathObject = super.getQueuePathObject(); + int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePathObject); showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath); if (maxAppsForReservation < 0) { maxAppsForReservation = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java new file mode 100644 index 0000000000000..9d15ca67e8288 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java @@ -0,0 +1,41 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; + +public final class QueuePrefixes { + + private QueuePrefixes() { + } + + public static String getQueuePrefix(QueuePath queuePath) { + String queueName = PREFIX + queuePath.getFullPath() + DOT; + return queueName; + } + + public static String getNodeLabelPrefix(QueuePath queuePath, String label) { + if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { + return getQueuePrefix(queuePath); + } + return getQueuePrefix(queuePath) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT; + } + + /** + * Get the auto created leaf queue's template configuration prefix + * Leaf queue's template capacities are configured at the parent queue + * + * @param queuePath parent queue's path + * @return Config prefix for leaf queue template configurations + */ + public static String getAutoCreatedQueueTemplateConfPrefix(QueuePath queuePath) { + return queuePath.getFullPath() + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX; + } + + public static QueuePath getAutoCreatedQueueObjectTemplateConfPrefix(QueuePath queuePath) { + return new QueuePath(queuePath.getFullPath(), AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java index a9d755227ffca..41c02204c852e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java @@ -24,7 +24,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_SETTINGS; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_WEIGHT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_WEIGHT_PATTERN; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix; public final class UserWeights { public static final float DEFAULT_WEIGHT = 1.0F; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java index 28eb33c553636..c2af5c5a576f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java @@ -21,6 +21,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.util.UnitsConversionUtil; import java.util.ArrayList; @@ -73,7 +74,7 @@ public QueueCapacityVector parse(CapacitySchedulerConfiguration conf, return QueueCapacityVector.of(100f, QueueCapacityType.PERCENTAGE); } - String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix( + String propertyName = QueuePrefixes.getNodeLabelPrefix( queuePath, label) + CapacitySchedulerConfiguration.CAPACITY; String capacityString = conf.get(propertyName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index bda69c1d0543e..c07297c9cff36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper; @XmlRootElement @@ -195,7 +196,7 @@ public static ArrayList getSortedQueueAclInfoList( String aclApplicationMaxPriority = "acl_" + StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString()); - String priorityAcls = conf.get(CapacitySchedulerConfiguration + String priorityAcls = conf.get(QueuePrefixes .getQueuePrefix(queuePath) + aclApplicationMaxPriority, CapacitySchedulerConfiguration.ALL_ACL); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java index f528a47449b0e..577dd519a0fa9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java @@ -19,6 +19,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -41,7 +42,7 @@ public LeafQueueTemplateInfo() { } // JAXB needs this public LeafQueueTemplateInfo(Configuration conf, String queuePath) { - String configPrefix = CapacitySchedulerConfiguration. + String configPrefix = QueuePrefixes. getQueuePrefix(queuePath) + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + DOT; conf.forEach(entry -> { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index be0dbf2a1ba41..b74fdd2e65108 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -116,7 +116,9 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getAutoCreatedQueueTemplateConfPrefix; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; @@ -321,8 +323,8 @@ public void testQueueSubmitWithACLsEnabledWithQueueMapping() CapacitySchedulerConfiguration(conf, false); csConf.set(PREFIX + "root.queues", "default,test"); - csConf.setCapacity("root.default", 50.0f); - csConf.setMaximumCapacity("root.default", 100.0f); + csConf.setCapacity(DEFAULT_QUEUE_PATH, 50.0f); + csConf.setMaximumCapacity(DEFAULT_QUEUE_PATH, 100.0f); csConf.setCapacity("root.test", 50.0f); csConf.setMaximumCapacity("root.test", 100.0f); @@ -414,21 +416,27 @@ public void testLegacyAutoCreatedQueuesWithACLTemplates() YarnConfiguration conf = createYarnACLEnabledConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration( conf, false); + + QueuePath parentQueuePath = new QueuePath("root.parent"); + QueuePath user1QueuePath = new QueuePath("root.parent.user1"); + QueuePath user2QueuePath = new QueuePath("root.parent.user2"); + csConf.set(PREFIX + "root.queues", "parent"); csConf.set(PREFIX + "root.acl_submit_applications", " "); csConf.set(PREFIX + "root.acl_administer_queue", " "); - csConf.setCapacity("root.parent", 100.0f); + csConf.setCapacity(parentQueuePath.getFullPath(), 100.0f); csConf.set(PREFIX + "root.parent.acl_administer_queue", "user1,user4"); csConf.set(PREFIX + "root.parent.acl_submit_applications", "user1,user4"); - csConf.setAutoCreateChildQueueEnabled("root.parent", true); - csConf.setAutoCreatedLeafQueueConfigCapacity("root.parent", 50f); - csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.parent", 100f); - csConf.set(getQueuePrefix(csConf.getAutoCreatedQueueTemplateConfPrefix("root.parent")) + - "acl_administer_queue", "user2,user4"); - csConf.set(getQueuePrefix(csConf.getAutoCreatedQueueTemplateConfPrefix("root.parent")) + - "acl_submit_applications", "user2,user4"); + csConf.setAutoCreateChildQueueEnabled(parentQueuePath.getFullPath(), true); + csConf.setAutoCreatedLeafQueueConfigCapacity(parentQueuePath.getFullPath(), 50f); + csConf.setAutoCreatedLeafQueueConfigMaxCapacity(parentQueuePath.getFullPath(), 100f); + String autoCreatedQueuePrefix = + getAutoCreatedQueueTemplateConfPrefix(parentQueuePath); + QueuePath autoCreatedQueuePath = new QueuePath(autoCreatedQueuePrefix); + csConf.set(getQueuePrefix(autoCreatedQueuePath) + "acl_administer_queue", "user2,user4"); + csConf.set(getQueuePrefix(autoCreatedQueuePath) + "acl_submit_applications", "user2,user4"); MockRM newMockRM = new MockRM(csConf); @@ -437,22 +445,22 @@ public void testLegacyAutoCreatedQueuesWithACLTemplates() // user1 has permission on root.parent so a queue would be created newMockRMContext.setQueuePlacementManager(createMockPlacementManager( - "user1", "user1", "root.parent")); + "user1", "user1", parentQueuePath.getFullPath())); verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(1)), newAppMonitor, newMockRMContext, "user1", - "root.parent.user1"); + user1QueuePath.getFullPath()); newMockRMContext.setQueuePlacementManager(createMockPlacementManager( - "user1|user2|user3|user4", "user2", "root.parent")); + "user1|user2|user3|user4", "user2", parentQueuePath.getFullPath())); // user2 has permission (due to ACL templates) verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(2)), newAppMonitor, newMockRMContext, "user2", - "root.parent.user2"); + user2QueuePath.getFullPath()); // user3 doesn't have permission verifyAppSubmissionFailure(newAppMonitor, @@ -464,12 +472,12 @@ public void testLegacyAutoCreatedQueuesWithACLTemplates() newAppMonitor, newMockRMContext, "user4", - "root.parent.user2"); + user2QueuePath.getFullPath()); // create the root.parent.user2 manually CapacityScheduler cs = ((CapacityScheduler) newMockRM.getResourceScheduler()); - cs.getCapacitySchedulerQueueManager().createQueue(new QueuePath("root.parent.user2")); + cs.getCapacitySchedulerQueueManager().createQueue(user2QueuePath); AutoCreatedLeafQueue autoCreatedLeafQueue = (AutoCreatedLeafQueue) cs.getQueue("user2"); Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedLeafQueue); ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue("parent"); @@ -482,7 +490,7 @@ public void testLegacyAutoCreatedQueuesWithACLTemplates() newAppMonitor, newMockRMContext, "user2", - "root.parent.user2"); + user2QueuePath.getFullPath()); // user3 doesn't have permission for root.parent.user2 queue verifyAppSubmissionFailure(newAppMonitor, @@ -494,7 +502,7 @@ public void testLegacyAutoCreatedQueuesWithACLTemplates() newAppMonitor, newMockRMContext, "user1", - "root.parent.user2"); + user2QueuePath.getFullPath()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java index 80f84c34503f9..4c261ab8aaaf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java @@ -55,6 +55,8 @@ import java.util.List; import java.util.Set; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_QUEUE_PATH; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils.toSet; import static org.junit.Assert.fail; @@ -265,15 +267,15 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - conf.setCapacity(A, 50); - conf.setMaximumCapacity(A, 100); + conf.setCapacity(A_QUEUE_PATH, 50); + conf.setMaximumCapacity(A_QUEUE_PATH, 100); conf.setAccessibleNodeLabels(A, toSet("x")); conf.setDefaultNodeLabelExpression(A, "x"); conf.setCapacityByLabel(A, "x", 100); final String B = CapacitySchedulerConfiguration.ROOT + ".b"; - conf.setCapacity(B, 50); - conf.setMaximumCapacity(B, 100); + conf.setCapacity(B_QUEUE_PATH, 50); + conf.setMaximumCapacity(B_QUEUE_PATH, 100); conf.setAccessibleNodeLabels(B, toSet("y")); conf.setDefaultNodeLabelExpression(B, "y"); conf.setCapacityByLabel(B, "y", 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java index 087b797f2d84c..db033a4be1de5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java @@ -65,8 +65,8 @@ public static void setMaxAllocMb(Configuration conf, int maxAllocMb) { } public static void setMaxAllocMb(CapacitySchedulerConfiguration conf, - String queueName, int maxAllocMb) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + QueuePath queuePath, int maxAllocMb) { + String propName = QueuePrefixes.getQueuePrefix(queuePath) + MAXIMUM_ALLOCATION_MB; conf.setInt(propName, maxAllocMb); } @@ -82,22 +82,22 @@ public static void setMaxAllocVcores(Configuration conf, int maxAllocVcores) { } public static void setMaxAllocVcores(CapacitySchedulerConfiguration conf, - String queueName, int maxAllocVcores) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + QueuePath queuePath, int maxAllocVcores) { + String propName = QueuePrefixes.getQueuePrefix(queuePath) + CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES; conf.setInt(propName, maxAllocVcores); } public static void setMaxAllocation(CapacitySchedulerConfiguration conf, - String queueName, String maxAllocation) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + QueuePath queuePath, String maxAllocation) { + String propName = QueuePrefixes.getQueuePrefix(queuePath) + MAXIMUM_ALLOCATION; conf.set(propName, maxAllocation); } public static void unsetMaxAllocation(CapacitySchedulerConfiguration conf, - String queueName) { - String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName) + QueuePath queuePath) { + String propName = QueuePrefixes.getQueuePrefix(queuePath) + MAXIMUM_ALLOCATION; conf.unset(propName); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java index 1adde7d8d196e..6c17345680d2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java @@ -27,6 +27,7 @@ public final class CapacitySchedulerQueueHelpers { + public static final String DEFAULT = CapacitySchedulerConfiguration.ROOT + ".default"; public static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; public static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; public static final String A_CHILD = A + ".a"; @@ -35,6 +36,15 @@ public final class CapacitySchedulerQueueHelpers { public static final String B1 = B + ".b1"; public static final String B2 = B + ".b2"; public static final String B3 = B + ".b3"; + public static final QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); + public static final QueuePath DEFAULT_QUEUE_PATH = new QueuePath(DEFAULT); + public static final QueuePath A_QUEUE_PATH = new QueuePath(A); + public static final QueuePath B_QUEUE_PATH = new QueuePath(B); + public static final QueuePath A1_QUEUE_PATH = new QueuePath(A1); + public static final QueuePath A2_QUEUE_PATH = new QueuePath(A2); + public static final QueuePath B1_QUEUE_PATH = new QueuePath(B1); + public static final QueuePath B2_QUEUE_PATH = new QueuePath(B2); + public static final QueuePath B3_QUEUE_PATH = new QueuePath(B3); public static final float A_CAPACITY = 10.5f; public static final float B_CAPACITY = 89.5f; public static final String P1 = CapacitySchedulerConfiguration.ROOT + ".p1"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index f06631d6c15d5..a2f7bac18b377 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -329,7 +329,7 @@ public void testLimitsComputation() throws Exception { // should return -1 if per queue setting not set assertEquals( (int)CapacitySchedulerConfiguration.UNDEFINED, - csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); + csConf.getMaximumApplicationsPerQueue(queue.getQueuePathObject())); int expectedMaxApps = (int) (CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * @@ -344,7 +344,7 @@ public void testLimitsComputation() throws Exception { // should default to global setting if per queue setting not set assertEquals(CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, csConf.getMaximumApplicationMasterResourcePerQueuePercent( - queue.getQueuePath()), epsilon); + queue.getQueuePathObject()), epsilon); // Change the per-queue max AM resources percentage. csConf.setFloat(PREFIX + queue.getQueuePath() @@ -364,7 +364,7 @@ public void testLimitsComputation() throws Exception { assertEquals(0.5f, csConf.getMaximumApplicationMasterResourcePerQueuePercent( - queue.getQueuePath()), epsilon); + queue.getQueuePathObject()), epsilon); assertThat(queue.calculateAndGetAMResourceLimit()).isEqualTo( Resource.newInstance(800 * GB, 1)); @@ -384,7 +384,8 @@ public void testLimitsComputation() throws Exception { clusterResource)); queue = (LeafQueue)queues.get(A); - assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); + assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue( + queue.getQueuePathObject())); assertEquals(9999, queue.getMaxApplications()); expectedMaxAppsPerUser = Math.min(9999, (int)(9999 * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index ef50e5271e718..0c78b427cc700 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -138,8 +138,9 @@ public void testAMResourceLimitWithLabels() throws Exception { // as 0.2 (Label X) and for Queue C1 as 0.2 (Empty Label) final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; final String C1 = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1"; + final QueuePath C1_QUEUE_PATH = new QueuePath(C1); config.setMaximumAMResourcePercentPerPartition(A1, "x", 0.2f); - config.setMaximumApplicationMasterResourcePerQueuePercent(C1, 0.2f); + config.setMaximumApplicationMasterResourcePerQueuePercent(C1_QUEUE_PATH, 0.2f); // Now inject node label manager with this updated config MockRM rm1 = new MockRM(config) { @@ -294,8 +295,9 @@ public void testAtleastOneAMRunPerPartition() throws Exception { // as 0.15 (Label X) and for Queue C1 as 0.15 (Empty Label) final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; final String C1 = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1"; + final QueuePath C1_QUEUE_PATH = new QueuePath(C1); config.setMaximumAMResourcePercentPerPartition(A1, "x", 0.15f); - config.setMaximumApplicationMasterResourcePerQueuePercent(C1, 0.15f); + config.setMaximumApplicationMasterResourcePerQueuePercent(C1_QUEUE_PATH, 0.15f); // inject node label manager MockRM rm1 = new MockRM(config) { @Override @@ -402,7 +404,8 @@ public void testDefaultAMLimitFromQueueForPartition() throws Exception { // After getting queue conf, configure AM resource percent for Queue A1 // as 0.2 (not for partition, rather in queue level) final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; - config.setMaximumApplicationMasterResourcePerQueuePercent(A1, 0.2f); + final QueuePath A1_QUEUE_PATH = new QueuePath(A1); + config.setMaximumApplicationMasterResourcePerQueuePercent(A1_QUEUE_PATH, 0.2f); // inject node label manager MockRM rm1 = new MockRM(config) { @Override @@ -627,8 +630,9 @@ public void testAMResourceLimitForMultipleApplications() throws Exception { */ final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; final String B1 = CapacitySchedulerConfiguration.ROOT + ".b" + ".b1"; + final QueuePath B1_QUEUE_PATH = new QueuePath(B1); config.setMaximumAMResourcePercentPerPartition(A1, "y", 0.25f); - config.setMaximumApplicationMasterResourcePerQueuePercent(B1, 0.15f); + config.setMaximumApplicationMasterResourcePerQueuePercent(B1_QUEUE_PATH, 0.15f); // Now inject node label manager with this updated config MockRM rm1 = new MockRM(config) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index 201214ec2674b..896abb8f6154d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -29,7 +29,11 @@ public class TestAutoCreatedQueueTemplate { private static final QueuePath TEST_QUEUE_AB = new QueuePath("root.a.b"); private static final QueuePath TEST_QUEUE_A = new QueuePath("root.a"); private static final QueuePath TEST_QUEUE_B = new QueuePath("root.b"); - private static final String ROOT = "root"; + private static final QueuePath TEST_QUEUE_WILDCARD = new QueuePath("*"); + private static final QueuePath TEST_QUEUE_ROOT_WILDCARD = new QueuePath("root.*"); + private static final QueuePath TEST_QUEUE_TWO_LEVEL_WILDCARDS = new QueuePath("root.*.*"); + private static final QueuePath TEST_QUEUE_A_WILDCARD = new QueuePath("root.a.*"); + private static final QueuePath ROOT = new QueuePath("root"); private CapacitySchedulerConfiguration conf; @@ -44,60 +48,60 @@ public void setUp() throws Exception { @Test public void testNonWildCardTemplate() { - conf.set(getTemplateKey(TEST_QUEUE_AB.getFullPath(), "capacity"), "6w"); + conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath()); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); Assert.assertEquals("weight is not set", 6f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); } @Test public void testOneLevelWildcardTemplate() { - conf.set(getTemplateKey("root.a.*", "capacity"), "6w"); + conf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "6w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath()); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); Assert.assertEquals("weight is not set", 6f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); } @Test public void testTwoLevelWildcardTemplate() { - conf.set(getTemplateKey("root.*", "capacity"), "6w"); - conf.set(getTemplateKey("root.*.*", "capacity"), "5w"); + conf.set(getTemplateKey(TEST_QUEUE_ROOT_WILDCARD, "capacity"), "6w"); + conf.set(getTemplateKey(TEST_QUEUE_TWO_LEVEL_WILDCARDS, "capacity"), "5w"); new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A) - .setTemplateEntriesForChild(conf, TEST_QUEUE_AB.getFullPath()); + .setTemplateEntriesForChild(conf, TEST_QUEUE_AB); new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB) - .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath()); + .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); Assert.assertEquals("weight is not set", 6f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_AB.getFullPath()), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_AB), 10e-6); Assert.assertEquals("weight is not set", 5f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); } @Test public void testIgnoredWhenRootWildcarded() { - conf.set(getTemplateKey("*", "capacity"), "6w"); + conf.set(getTemplateKey(TEST_QUEUE_WILDCARD, "capacity"), "6w"); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT)); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath()); + new AutoCreatedQueueTemplate(conf, ROOT); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); Assert.assertEquals("weight is set", -1f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); } @Test public void testIgnoredWhenNoParent() { - conf.set(getTemplateKey("root", "capacity"), "6w"); + conf.set(getTemplateKey(ROOT, "capacity"), "6w"); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT)); + new AutoCreatedQueueTemplate(conf, ROOT); template.setTemplateEntriesForChild(conf, ROOT); Assert.assertEquals("weight is set", -1f, @@ -106,10 +110,10 @@ public void testIgnoredWhenNoParent() { @Test public void testWildcardAfterRoot() { - conf.set(getTemplateKey("root.*", "acl_submit_applications"), "user"); + conf.set(getTemplateKey(TEST_QUEUE_ROOT_WILDCARD, "acl_submit_applications"), "user"); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, new QueuePath("root.a")); - template.setTemplateEntriesForChild(conf, "root.a"); + new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); Assert.assertEquals("acl_submit_applications is set", "user", template.getTemplateProperties().get("acl_submit_applications")); @@ -117,53 +121,53 @@ public void testWildcardAfterRoot() { @Test public void testTemplatePrecedence() { - conf.set(getTemplateKey("root.a.b", "capacity"), "6w"); - conf.set(getTemplateKey("root.a.*", "capacity"), "4w"); - conf.set(getTemplateKey("root.*.*", "capacity"), "2w"); + conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w"); + conf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "4w"); + conf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "2w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath()); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); Assert.assertEquals( "explicit template does not have the highest precedence", 6f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); CapacitySchedulerConfiguration newConf = new CapacitySchedulerConfiguration(); - newConf.set(getTemplateKey("root.a.*", "capacity"), "4w"); + newConf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "4w"); template = new AutoCreatedQueueTemplate(newConf, TEST_QUEUE_AB); - template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC.getFullPath()); + template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC); Assert.assertEquals("precedence is invalid", 4f, - newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); + newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); } @Test public void testRootTemplate() { - conf.set(getTemplateKey("root", "capacity"), "2w"); + conf.set(getTemplateKey(ROOT, "capacity"), "2w"); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT)); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath()); + new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); Assert.assertEquals("root property is not set", 2f, - conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6); + conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); } @Test public void testQueueSpecificTemplates() { - conf.set(getTemplateKey("root", "capacity"), "2w"); - conf.set(getLeafTemplateKey("root", + conf.set(getTemplateKey(ROOT, "capacity"), "2w"); + conf.set(getLeafTemplateKey(ROOT, "default-node-label-expression"), "test"); - conf.set(getLeafTemplateKey("root", "capacity"), "10w"); + conf.set(getLeafTemplateKey(ROOT, "capacity"), "10w"); conf.setBoolean(getParentTemplateKey( - "root", AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE), false); + ROOT, AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE), false); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT)); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath()); - template.setTemplateEntriesForChild(conf, TEST_QUEUE_B.getFullPath(), true); + new AutoCreatedQueueTemplate(conf, ROOT); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_B, true); Assert.assertNull("default-node-label-expression is set for parent", conf.getDefaultNodeLabelExpression(TEST_QUEUE_A.getFullPath())); @@ -173,24 +177,24 @@ public void testQueueSpecificTemplates() { conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A.getFullPath())); Assert.assertEquals("weight should not be overridden when set by " + "queue type specific template", - 10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B.getFullPath()), 10e-6); + 10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B), 10e-6); Assert.assertEquals("weight should be set by common template", - 2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6); + 2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); } - private String getTemplateKey(String queuePath, String entryKey) { - return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + private String getTemplateKey(QueuePath queuePath, String entryKey) { + return QueuePrefixes.getQueuePrefix(queuePath) + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey; } - private String getParentTemplateKey(String queuePath, String entryKey) { - return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + private String getParentTemplateKey(QueuePath queuePath, String entryKey) { + return QueuePrefixes.getQueuePrefix(queuePath) + AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + entryKey; } - private String getLeafTemplateKey(String queuePath, String entryKey) { - return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + private String getLeafTemplateKey(QueuePath queuePath, String entryKey) { + return QueuePrefixes.getQueuePrefix(queuePath) + AutoCreatedQueueTemplate.AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + entryKey; } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java index 5b233f27a8034..54a75200d4be6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java @@ -117,7 +117,7 @@ public void testCapacitySchedulerJobWhenConfigureCustomResourceType() .getConfigurationWithMultipleQueues(conf); newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class, ResourceCalculator.class); - newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a") + newConf.set(QueuePrefixes.getQueuePrefix("root.a") + MAXIMUM_ALLOCATION_MB, "4096"); // We must set this to false to avoid MockRM init configuration with // resource-types.xml by ResourceUtils.resetResourceTypes(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 7c407dfd4ad4b..6a712027dd68d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -23,20 +23,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocVcores; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMinAllocMb; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMinAllocVcores; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.findQueue; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupAdditionalQueues; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupBlockedQueueConfiguration; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupOtherBlockedQueueConfiguration; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkApplicationResourceUsage; @@ -604,8 +591,8 @@ public void testQueueMaximumAllocations() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); - setMaxAllocMb(conf, A1, 1024); - setMaxAllocVcores(conf, A1, 1); + setMaxAllocMb(conf, A1_QUEUE_PATH, 1024); + setMaxAllocVcores(conf, A1_QUEUE_PATH, 1); scheduler.init(conf); scheduler.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java index d192e7dcc6933..ac4be49359efd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java @@ -1042,7 +1042,7 @@ public void testMaxParallelAppsPendingQueueMetrics() throws Exception { MockRM rm = setUpMove(); ResourceScheduler scheduler = rm.getResourceScheduler(); CapacityScheduler cs = (CapacityScheduler) scheduler; - cs.getQueueContext().getConfiguration().setInt(CapacitySchedulerConfiguration.getQueuePrefix(A1) + cs.getQueueContext().getConfiguration().setInt(QueuePrefixes.getQueuePrefix(A1) + CapacitySchedulerConfiguration.MAX_PARALLEL_APPLICATIONS, MAX_PARALLEL_APPS); cs.reinitialize(cs.getQueueContext().getConfiguration(), mockContext); List attemptIds = new ArrayList<>(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index 5df92e20a276f..a883182a283e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -782,7 +782,7 @@ protected void validateEffectiveMinResource(ResourceManager rm, if (expectedQueueEntitlements.get(label).getCapacity() > EPSILON) { if (leafQueue.getCapacityConfigType().equals(ABSOLUTE_RESOURCE)) { - String templatePrefix = cs.getConfiguration().getAutoCreatedQueueTemplateConfPrefix( + String templatePrefix = QueuePrefixes.getAutoCreatedQueueTemplateConfPrefix( parentQueue.getQueuePath()); Resource resourceTemplate = parentQueue.getLeafQueueTemplate().getLeafQueueConfigs() .getMinimumResourceRequirement(label, templatePrefix, RESOURCE_TYPES); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java index 2cbb11fc9bdbe..b1534e3f96593 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java @@ -25,7 +25,6 @@ import java.util.Set; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -57,7 +56,7 @@ private void setSubmitAppsConfig(CapacitySchedulerConfiguration csConf, String q } private String getSubmitAppsConfigKey(String queue) { - return getQueuePrefix(queue) + "acl_submit_applications"; + return QueuePrefixes.getQueuePrefix(queue) + "acl_submit_applications"; } private void testWithGivenAclNoOneHasAccess(String queue, String aclValue) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java index 48e8569f0f7c5..b2b304d4f8e28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.waitforNMRegistered; @@ -200,7 +201,7 @@ public void testExcessReservationWillBeUnreserved() throws Exception { YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + ".resource-based.sorting-interval.ms", 0); - newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default", + newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, 1.0f); MockRM rm1 = new MockRM(newConf); @@ -286,7 +287,7 @@ public void testAllocateForReservedContainer() throws Exception { YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + ".resource-based.sorting-interval.ms", 0); - newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default", + newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, 1.0f); MockRM rm1 = new MockRM(newConf); @@ -357,7 +358,7 @@ public void testAllocateOfReservedContainerFromAnotherNode() YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + ".resource-based.sorting-interval.ms", 0); - newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default", + newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, 1.0f); MockRM rm1 = new MockRM(newConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index 037312b716a05..b77ae2f49c09a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -50,6 +50,8 @@ import java.io.IOException; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; + public class TestCapacitySchedulerNewQueueAutoCreation extends TestCapacitySchedulerAutoCreatedQueueBase { private static final Logger LOG = LoggerFactory.getLogger( @@ -63,6 +65,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation private CapacitySchedulerQueueManager autoQueueHandler; private AutoCreatedQueueDeletionPolicy policy = new AutoCreatedQueueDeletionPolicy(); + private QueuePath queuePath public CapacityScheduler getCs() { return cs; @@ -88,11 +91,11 @@ public void setUp() throws Exception { // By default, set 3 queues, a/b, and a.a1 csConf.setQueues("root", new String[]{"a", "b"}); - csConf.setNonLabeledQueueWeight("root", 1f); - csConf.setNonLabeledQueueWeight("root.a", 1f); - csConf.setNonLabeledQueueWeight("root.b", 1f); + csConf.setNonLabeledQueueWeight(ROOT_QUEUE_PATH, 1f); + csConf.setNonLabeledQueueWeight(A_QUEUE_PATH, 1f); + csConf.setNonLabeledQueueWeight(B_QUEUE_PATH, 1f); csConf.setQueues("root.a", new String[]{"a1"}); - csConf.setNonLabeledQueueWeight("root.a.a1", 1f); + csConf.setNonLabeledQueueWeight(A1_QUEUE_PATH, 1f); csConf.setAutoQueueCreationV2Enabled("root", true); csConf.setAutoQueueCreationV2Enabled("root.a", true); csConf.setAutoQueueCreationV2Enabled("root.e", true); @@ -327,7 +330,7 @@ public void testAutoCreateMaximumQueueDepth() () -> createQueue("root.a.a3-auto.a4-auto.a5-auto")); // Set depth 3 for root.a, making it a valid scenario - csConf.setMaximumAutoCreatedQueueDepth("root.a", 3); + csConf.setMaximumAutoCreatedQueueDepth(A_QUEUE_PATH, 3); cs.reinitialize(csConf, mockRM.getRMContext()); try { createQueue("root.a.a3-auto.a4-auto.a5-auto"); @@ -338,7 +341,7 @@ public void testAutoCreateMaximumQueueDepth() // Set global depth to 3 csConf.setMaximumAutoCreatedQueueDepth(3); - csConf.unset(CapacitySchedulerConfiguration.getQueuePrefix("root.a") + csConf.unset(QueuePrefixes.getQueuePrefix(A_QUEUE_PATH) + CapacitySchedulerConfiguration.MAXIMUM_QUEUE_DEPTH); cs.reinitialize(csConf, mockRM.getRMContext()); try { @@ -349,7 +352,7 @@ public void testAutoCreateMaximumQueueDepth() } // Set depth on a dynamic queue, which has no effect on auto queue creation validation - csConf.setMaximumAutoCreatedQueueDepth("root.a.a6-auto.a7-auto.a8-auto", 10); + csConf.setMaximumAutoCreatedQueueDepth(new QueuePath("root.a.a6-auto.a7-auto.a8-auto"), 10); Assert.assertThrows(SchedulerDynamicEditException.class, () -> createQueue("root.a.a6-auto.a7-auto.a8-auto.a9-auto.a10-auto.a11-auto")); } @@ -386,7 +389,7 @@ public void testConvertDynamicToStaticQueue() throws Exception { createBasicQueueStructureAndValidate(); // Now, update root.a's weight to 6 - csConf.setNonLabeledQueueWeight("root.a", 6f); + csConf.setNonLabeledQueueWeight(new QueuePath("root.a"), 6f); cs.reinitialize(csConf, mockRM.getRMContext()); // Double confirm, after refresh, we should still see root queue has 5 @@ -405,7 +408,7 @@ public void testConvertDynamicToStaticQueue() throws Exception { // Set queue c-auto's weight to 6, and mark c-auto to be static queue csConf.setQueues("root", new String[]{"a", "b", "c-auto"}); - csConf.setNonLabeledQueueWeight("root.c-auto", 6f); + csConf.setNonLabeledQueueWeight(new QueuePath("root.c-auto"), 6f); cs.reinitialize(csConf, mockRM.getRMContext()); // Get queue c @@ -423,9 +426,9 @@ public void testConvertDynamicToStaticQueue() throws Exception { // Do change 2nd level queue from dynamic to static csConf.setQueues("root", new String[]{"a", "b", "c-auto", "e-auto"}); - csConf.setNonLabeledQueueWeight("root.e-auto", 6f); + csConf.setNonLabeledQueueWeight(new QueuePath("root.e-auto"), 6f); csConf.setQueues("root.e-auto", new String[]{"e1-auto"}); - csConf.setNonLabeledQueueWeight("root.e-auto.e1-auto", 6f); + csConf.setNonLabeledQueueWeight(new QueuePath("root.e-auto.e1-auto"), 6f); cs.reinitialize(csConf, mockRM.getRMContext()); // Get queue e1 @@ -453,8 +456,8 @@ public void testConvertDynamicParentToStaticParent() throws Exception { startScheduler(); createQueue("root.d-auto.d1-auto"); csConf.setQueues("root", new String[]{"a", "b", "d-auto"}); - csConf.setNonLabeledQueueWeight("root.a", 6f); - csConf.setNonLabeledQueueWeight("root.d-auto", 1f); + csConf.setNonLabeledQueueWeight(new QueuePath("root.a"), 6f); + csConf.setNonLabeledQueueWeight(new QueuePath("root.d-auto"), 1f); cs.reinitialize(csConf, mockRM.getRMContext()); CSQueue d = cs.getQueue("root.d-auto"); @@ -490,11 +493,11 @@ public void testChildlessParentQueueWhenAutoQueueCreationEnabled() throws Exception { startScheduler(); csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"}); - csConf.setNonLabeledQueueWeight("root", 1f); - csConf.setNonLabeledQueueWeight("root.a", 1f); - csConf.setNonLabeledQueueWeight("root.b", 1f); + csConf.setNonLabeledQueueWeight(ROOT_QUEUE_PATH, 1f); + csConf.setNonLabeledQueueWeight(A_QUEUE_PATH, 1f); + csConf.setNonLabeledQueueWeight(B_QUEUE_PATH, 1f); csConf.setQueues("root.a", new String[]{"a1"}); - csConf.setNonLabeledQueueWeight("root.a.a1", 1f); + csConf.setNonLabeledQueueWeight(A1_QUEUE_PATH, 1f); csConf.setAutoQueueCreationV2Enabled("root", true); csConf.setAutoQueueCreationV2Enabled("root.a", true); cs.reinitialize(csConf, mockRM.getRMContext()); @@ -505,11 +508,11 @@ public void testChildlessParentQueueWhenAutoQueueCreationEnabled() empty.stopQueue(); csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"}); - csConf.setNonLabeledQueueWeight("root", 1f); - csConf.setNonLabeledQueueWeight("root.a", 1f); - csConf.setNonLabeledQueueWeight("root.b", 1f); + csConf.setNonLabeledQueueWeight(ROOT_QUEUE_PATH, 1f); + csConf.setNonLabeledQueueWeight(A_QUEUE_PATH, 1f); + csConf.setNonLabeledQueueWeight(B_QUEUE_PATH, 1f); csConf.setQueues("root.a", new String[]{"a1"}); - csConf.setNonLabeledQueueWeight("root.a.a1", 1f); + csConf.setNonLabeledQueueWeight(A1_QUEUE_PATH, 1f); csConf.setAutoQueueCreationV2Enabled("root", true); csConf.setAutoQueueCreationV2Enabled("root.a", true); csConf.setAutoQueueCreationV2Enabled("root.empty-auto-parent", true); @@ -629,7 +632,7 @@ public void testAutoQueueCreationMaxAppUpdate() throws Exception { Assert.assertEquals(1000, e.getMaxApplications()); // when set some queue for max apps - csConf.setMaximumApplicationsPerQueue("root.e1", 50); + csConf.setMaximumApplicationsPerQueue(new QueuePath("root.e1"), 50); createQueue("root.e1"); LeafQueue e1 = (LeafQueue)cs. getQueue("root.e1"); @@ -710,8 +713,13 @@ public void testAutoCreateQueueMaxQueuesLimit() throws Exception { @Test public void testAutoCreatedQueueTemplateConfig() throws Exception { startScheduler(); + + QueuePath childQueuesOfA = new QueuePath("root.a.*"); + QueuePath aQueuePath = new QueuePath("root.a"); + QueuePath cQueuePath = new QueuePath("root.c"); + csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( - "root.a.*") + "capacity", "6w"); + childQueuesOfA) + "capacity", "6w"); cs.reinitialize(csConf, mockRM.getRMContext()); AbstractLeafQueue a2 = createQueue("root.a.a-auto.a2"); @@ -725,9 +733,9 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { // Set the user-limit-factor and maximum-am-resource-percent via templates to ensure their // modified defaults are indeed overridden csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( - "root.a.*") + "user-limit-factor", "10"); + childQueuesOfA) + "user-limit-factor", "10"); csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( - "root.a.*") + "maximum-am-resource-percent", "0.8"); + childQueuesOfA) + "maximum-am-resource-percent", "0.8"); cs.reinitialize(csConf, mockRM.getRMContext()); a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2"); @@ -739,13 +747,13 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { 0.8f, a2.getMaxAMResourcePerQueuePercent(), 1e-6); - csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f); + csConf.setNonLabeledQueueWeight(new QueuePath("root.a.a-auto.a2"), 4f); cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals("weight is not explicitly set", 4f, a2.getQueueCapacities().getWeight(), 1e-6); csConf.setBoolean(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( - "root.a") + CapacitySchedulerConfiguration + aQueuePath) + CapacitySchedulerConfiguration .AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE, false); cs.reinitialize(csConf, mockRM.getRMContext()); AbstractLeafQueue a3 = createQueue("root.a.a3"); @@ -754,7 +762,7 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { // Set the capacity of label TEST csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( - "root.c") + "accessible-node-labels.TEST.capacity", "6w"); + cQueuePath) + "accessible-node-labels.TEST.capacity", "6w"); csConf.setQueues("root", new String[]{"a", "b", "c"}); csConf.setAutoQueueCreationV2Enabled("root.c", true); cs.reinitialize(csConf, mockRM.getRMContext()); @@ -771,7 +779,7 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { public void testAutoCreatedQueueConfigChange() throws Exception { startScheduler(); AbstractLeafQueue a2 = createQueue("root.a.a-auto.a2"); - csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f); + csConf.setNonLabeledQueueWeight(a2.getQueuePathObject(), 4f); cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals("weight is not explicitly set", 4f, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java index 27eb3ac7bfd55..b89cae9b7ac0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java @@ -23,6 +23,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerOvercommit; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; + /** * Test changing resources and overcommit in the Capacity Scheduler * {@link CapacityScheduler}. @@ -39,13 +42,13 @@ protected Configuration getConfiguration() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); csConf.setMaximumApplicationMasterResourcePerQueuePercent( - CapacitySchedulerConfiguration.ROOT, 100.0f); + ROOT_QUEUE_PATH, 100.0f); csConf.setMaximumAMResourcePercentPerPartition( CapacitySchedulerConfiguration.ROOT, "", 100.0f); csConf.setMaximumApplicationMasterResourcePerQueuePercent( - CapacitySchedulerConfiguration.ROOT + ".default", 100.0f); + DEFAULT_QUEUE_PATH, 100.0f); csConf.setMaximumAMResourcePercentPerPartition( - CapacitySchedulerConfiguration.ROOT + ".default", "", 100.0f); + DEFAULT_QUEUE_PATH.getFullPath(), "", 100.0f); return csConf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java index b8209a54952e7..ace82932e983f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java @@ -62,6 +62,7 @@ import java.util.concurrent.atomic.AtomicLong; import static org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -448,7 +449,7 @@ CapacitySchedulerConfiguration createCSConfWithManyQueues(int numQueues) CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(); csconf.setResourceComparator(DominantResourceCalculator.class); - csconf.setMaximumApplicationMasterResourcePerQueuePercent("root", 100.0f); + csconf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT_QUEUE_PATH, 100.0f); csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f); csconf.setCapacity("root.default", 0.0f); csconf.setOffSwitchPerHeartbeatLimit(numQueues); @@ -457,14 +458,14 @@ CapacitySchedulerConfiguration createCSConfWithManyQueues(int numQueues) String[] subQueues = new String[numQueues]; for (int i = 0; i < numQueues; i++) { String queueName = String.format("%03d", i); - String queuePath = "root." + queueName; + QueuePath queuePath = new QueuePath("root." + queueName); subQueues[i] = queueName; csconf.setMaximumApplicationMasterResourcePerQueuePercent( queuePath, 100.0f); - csconf.setMaximumAMResourcePercentPerPartition(queuePath, "", 100.0f); - csconf.setCapacity(queuePath, capacity); - csconf.setUserLimitFactor(queuePath, 100.0f); - csconf.setMaximumCapacity(queuePath, 100.0f); + csconf.setMaximumAMResourcePercentPerPartition(queuePath.getFullPath(), "", 100.0f); + csconf.setCapacity(queuePath.getFullPath(), capacity); + csconf.setUserLimitFactor(queuePath.getFullPath(), 100.0f); + csconf.setMaximumCapacity(queuePath.getFullPath(), 100.0f); } csconf.setQueues("root", subQueues); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java index fc1870097e9a4..a81684a662f07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java @@ -43,25 +43,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocVcores; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocation; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.unsetMaxAllocation; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ExpectedCapacities; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.findQueue; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.getDefaultCapacities; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfWithoutChildrenOfB; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithB1AsParentQueue; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createMockRMContext; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager; @@ -219,7 +201,7 @@ public void testRefreshQueuesMaxAllocationRefresh() throws Exception { assertEquals("queue max allocation", ((LeafQueue) queueA1) .getMaximumAllocation().getMemorySize(), 8192); - setMaxAllocMb(conf, A1, 4096); + setMaxAllocMb(conf, A1_QUEUE_PATH, 4096); try { cs.reinitialize(conf, mockContext); @@ -229,10 +211,10 @@ public void testRefreshQueuesMaxAllocationRefresh() throws Exception { e.getCause().toString().contains("not be decreased")); } - setMaxAllocMb(conf, A1, 8192); + setMaxAllocMb(conf, A1_QUEUE_PATH, 8192); cs.reinitialize(conf, mockContext); - setMaxAllocVcores(conf, A1, + setMaxAllocVcores(conf, A1_QUEUE_PATH, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES - 1); try { cs.reinitialize(conf, mockContext); @@ -254,7 +236,7 @@ public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception { cs.init(conf); cs.start(); // change max allocation for B3 queue to be larger then cluster max - setMaxAllocMb(conf, B3, + setMaxAllocMb(conf, B3_QUEUE_PATH, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 2048); try { cs.reinitialize(conf, mockContext); @@ -264,11 +246,11 @@ public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception { e.getCause().getMessage().contains("maximum allocation")); } - setMaxAllocMb(conf, B3, + setMaxAllocMb(conf, B3_QUEUE_PATH, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); cs.reinitialize(conf, mockContext); - setMaxAllocVcores(conf, B3, + setMaxAllocVcores(conf, B3_QUEUE_PATH, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1); try { cs.reinitialize(conf, mockContext); @@ -291,8 +273,8 @@ public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); setMaxAllocVcores(conf, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); - setMaxAllocMb(conf, A1, 4096); - setMaxAllocVcores(conf, A1, 2); + setMaxAllocMb(conf, A1_QUEUE_PATH, 4096); + setMaxAllocVcores(conf, A1_QUEUE_PATH, 2); cs.init(conf); cs.start(); cs.reinitialize(conf, mockContext); @@ -324,8 +306,8 @@ public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { assertEquals("queue max allocation", 4096, queueA1.getMaximumAllocation().getMemorySize()); - setMaxAllocMb(conf, A1, 6144); - setMaxAllocVcores(conf, A1, 3); + setMaxAllocMb(conf, A1_QUEUE_PATH, 6144); + setMaxAllocVcores(conf, A1_QUEUE_PATH, 3); cs.reinitialize(conf, null); // conf will have changed but we shouldn't be able to change max allocation // for the actual queue @@ -362,8 +344,8 @@ public void testRefreshQueuesMaxAllocationCSError() throws Exception { setupQueueConfiguration(conf); setMaxAllocMb(conf, 10240); setMaxAllocVcores(conf, 10); - setMaxAllocMb(conf, A1, 4096); - setMaxAllocVcores(conf, A1, 4); + setMaxAllocMb(conf, A1_QUEUE_PATH, 4096); + setMaxAllocVcores(conf, A1_QUEUE_PATH, 4); cs.init(conf); cs.start(); cs.reinitialize(conf, mockContext); @@ -408,8 +390,8 @@ public void testRefreshQueuesMaxAllocationCSLarger() throws Exception { setupQueueConfiguration(conf); setMaxAllocMb(conf, 10240); setMaxAllocVcores(conf, 10); - setMaxAllocMb(conf, A1, 4096); - setMaxAllocVcores(conf, A1, 4); + setMaxAllocMb(conf, A1_QUEUE_PATH, 4096); + setMaxAllocVcores(conf, A1_QUEUE_PATH, 4); cs.init(conf); cs.start(); cs.reinitialize(conf, mockContext); @@ -733,11 +715,11 @@ public void testQueuesMaxAllocationInheritance() throws Exception { YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); // Test the child queue overrides - setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, + setMaxAllocation(conf, ROOT_QUEUE_PATH, "memory-mb=4096,vcores=2"); - setMaxAllocation(conf, A1, "memory-mb=6144,vcores=2"); - setMaxAllocation(conf, B, "memory-mb=5120, vcores=2"); - setMaxAllocation(conf, B2, "memory-mb=1024, vcores=2"); + setMaxAllocation(conf, A1_QUEUE_PATH, "memory-mb=6144,vcores=2"); + setMaxAllocation(conf, B_QUEUE_PATH, "memory-mb=5120, vcores=2"); + setMaxAllocation(conf, B2_QUEUE_PATH, "memory-mb=1024, vcores=2"); cs.init(conf); cs.start(); @@ -777,12 +759,12 @@ public void testQueuesMaxAllocationInheritance() throws Exception { queueB2.getMaximumAllocation().getMemorySize()); // Test get the max-allocation from different parent - unsetMaxAllocation(conf, A1); - unsetMaxAllocation(conf, B); - unsetMaxAllocation(conf, B1); - setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, + unsetMaxAllocation(conf, A1_QUEUE_PATH); + unsetMaxAllocation(conf, B_QUEUE_PATH); + unsetMaxAllocation(conf, B1_QUEUE_PATH); + setMaxAllocation(conf, ROOT_QUEUE_PATH, "memory-mb=6144,vcores=2"); - setMaxAllocation(conf, A, "memory-mb=8192,vcores=2"); + setMaxAllocation(conf, A_QUEUE_PATH, "memory-mb=8192,vcores=2"); cs.reinitialize(conf, mockContext); @@ -806,9 +788,9 @@ public void testQueuesMaxAllocationInheritance() throws Exception { queueB1.getMaximumAllocation().getVirtualCores()); // Test the default - unsetMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT); - unsetMaxAllocation(conf, A); - unsetMaxAllocation(conf, A1); + unsetMaxAllocation(conf, ROOT_QUEUE_PATH); + unsetMaxAllocation(conf, A_QUEUE_PATH); + unsetMaxAllocation(conf, A1_QUEUE_PATH); cs.reinitialize(conf, mockContext); assertEquals("max capability MB in CS", @@ -855,7 +837,7 @@ public void testVerifyQueuesMaxAllocationConf() throws Exception { cs.reinitialize(conf, mockContext); checkQueueStructureCapacities(cs); - setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, + setMaxAllocation(conf, ROOT_QUEUE_PATH, "memory-mb=" + largerMem + ",vcores=2"); try { cs.reinitialize(conf, mockContext); @@ -865,10 +847,10 @@ public void testVerifyQueuesMaxAllocationConf() throws Exception { e.getCause().getMessage().contains("maximum allocation")); } - setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT, + setMaxAllocation(conf, ROOT_QUEUE_PATH, "memory-mb=4096,vcores=2"); - setMaxAllocation(conf, A, "memory-mb=6144,vcores=2"); - setMaxAllocation(conf, A1, "memory-mb=" + largerMem + ",vcores=2"); + setMaxAllocation(conf, A_QUEUE_PATH, "memory-mb=6144,vcores=2"); + setMaxAllocation(conf, A1_QUEUE_PATH, "memory-mb=" + largerMem + ",vcores=2"); try { cs.reinitialize(conf, mockContext); fail("Queue A1 maximum allocation can't exceed the cluster setting"); @@ -876,7 +858,7 @@ public void testVerifyQueuesMaxAllocationConf() throws Exception { assertTrue("maximum allocation exception", e.getCause().getMessage().contains("maximum allocation")); } - setMaxAllocation(conf, A1, "memory-mb=8192" + ",vcores=" + largerVcores); + setMaxAllocation(conf, A1_QUEUE_PATH, "memory-mb=8192" + ",vcores=" + largerVcores); try { cs.reinitialize(conf, mockContext); fail("Queue A1 maximum allocation can't exceed the cluster setting"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java index 115813a01717f..2441a9627822a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java @@ -51,6 +51,8 @@ import java.util.ArrayList; import java.util.Set; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; + public class TestCapacitySchedulerWeightMode { private static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; private static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; @@ -116,41 +118,41 @@ public static Configuration getCSConfWithQueueLabelsWeightOnly( // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", "b" }); - conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); - conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); + conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "x", 100); + conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "y", 100); + conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "z", 100); - conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1); + conf.setLabeledQueueWeight(A_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(A, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); - conf.setLabeledQueueWeight(A, "x", 100); - conf.setLabeledQueueWeight(A, "y", 50); + conf.setLabeledQueueWeight(A_QUEUE_PATH, "x", 100); + conf.setLabeledQueueWeight(A_QUEUE_PATH, "y", 50); - conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9); + conf.setLabeledQueueWeight(B_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 9); conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); - conf.setLabeledQueueWeight(B, "y", 50); - conf.setLabeledQueueWeight(B, "z", 100); + conf.setLabeledQueueWeight(B_QUEUE_PATH, "y", 50); + conf.setLabeledQueueWeight(B_QUEUE_PATH, "z", 100); // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); - conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 100); + conf.setLabeledQueueWeight(A1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 100); conf.setMaximumCapacity(A1, 100); conf.setAccessibleNodeLabels(A1, toSet("x", "y")); conf.setDefaultNodeLabelExpression(A1, "x"); - conf.setLabeledQueueWeight(A1, "x", 100); - conf.setLabeledQueueWeight(A1, "y", 100); + conf.setLabeledQueueWeight(A1_QUEUE_PATH, "x", 100); + conf.setLabeledQueueWeight(A1_QUEUE_PATH, "y", 100); conf.setQueues(B, new String[] { "b1", "b2" }); - conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 50); + conf.setLabeledQueueWeight(B1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 50); conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); - conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 50); + conf.setLabeledQueueWeight(B2_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 50); conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); - conf.setLabeledQueueWeight(B2, "y", 100); - conf.setLabeledQueueWeight(B2, "z", 100); + conf.setLabeledQueueWeight(B2_QUEUE_PATH, "y", 100); + conf.setLabeledQueueWeight(B2_QUEUE_PATH, "z", 100); return conf; } @@ -175,21 +177,21 @@ public static Configuration getCSConfWithLabelsParentUseWeightChildUsePct( // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", "b" }); - conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); - conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); + conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "x", 100); + conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "y", 100); + conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "z", 100); - conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1); + conf.setLabeledQueueWeight(A_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(A, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); - conf.setLabeledQueueWeight(A, "x", 100); - conf.setLabeledQueueWeight(A, "y", 50); + conf.setLabeledQueueWeight(A_QUEUE_PATH, "x", 100); + conf.setLabeledQueueWeight(A_QUEUE_PATH, "y", 50); - conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9); + conf.setLabeledQueueWeight(B_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 9); conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); - conf.setLabeledQueueWeight(B, "y", 50); - conf.setLabeledQueueWeight(B, "z", 100); + conf.setLabeledQueueWeight(B_QUEUE_PATH, "y", 50); + conf.setLabeledQueueWeight(B_QUEUE_PATH, "z", 100); // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); @@ -252,23 +254,23 @@ public static Configuration getCSConfWithLabelsParentUsePctChildUseWeight( // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); - conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 1); + conf.setLabeledQueueWeight(A1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(A1, 100); conf.setAccessibleNodeLabels(A1, toSet("x", "y")); conf.setDefaultNodeLabelExpression(A1, "x"); - conf.setLabeledQueueWeight(A1, "x", 1); - conf.setLabeledQueueWeight(A1, "y", 1); + conf.setLabeledQueueWeight(A1_QUEUE_PATH, "x", 1); + conf.setLabeledQueueWeight(A1_QUEUE_PATH, "y", 1); conf.setQueues(B, new String[] { "b1", "b2" }); - conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 1); + conf.setLabeledQueueWeight(B1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); - conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 1); + conf.setLabeledQueueWeight(B2_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); - conf.setLabeledQueueWeight(B2, "y", 1); - conf.setLabeledQueueWeight(B2, "z", 1); + conf.setLabeledQueueWeight(B2_QUEUE_PATH, "y", 1); + conf.setLabeledQueueWeight(B2_QUEUE_PATH, "z", 1); return conf; } @@ -424,9 +426,9 @@ public void testQueueInfoWeight() throws Exception { conf); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "default"}); - csConf.setNonLabeledQueueWeight("root.a", 1); - csConf.setNonLabeledQueueWeight("root.b", 2); - csConf.setNonLabeledQueueWeight("root.default", 3); + csConf.setNonLabeledQueueWeight(A_QUEUE_PATH, 1); + csConf.setNonLabeledQueueWeight(B_QUEUE_PATH, 2); + csConf.setNonLabeledQueueWeight(DEFAULT_QUEUE_PATH, 3); // Check queue info capacity CapacityScheduler cs = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index 98cffda51cc81..78acb5e5f4232 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -57,6 +57,9 @@ import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Assert; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; import static org.junit.Assert.assertEquals; import org.junit.Test; @@ -116,9 +119,9 @@ public void testMaximumAllocationRefreshWithMultipleResourceTypes() throws Excep CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(); - csconf.setMaximumApplicationMasterResourcePerQueuePercent("root", 100.0f); + csconf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT_QUEUE_PATH, 100.0f); csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f); - csconf.setMaximumApplicationMasterResourcePerQueuePercent("root.default", + csconf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, 100.0f); csconf.setMaximumAMResourcePercentPerPartition("root.default", "", 100.0f); csconf.setResourceComparator(DominantResourceCalculator.class); @@ -271,9 +274,9 @@ public void testMaxLimitsOfQueueWithMultipleResources() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setMaximumApplicationMasterResourcePerQueuePercent("root", 100.0f); + csConf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT_QUEUE_PATH, 100.0f); csConf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f); - csConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default", + csConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, 100.0f); csConf.setMaximumAMResourcePercentPerPartition("root.default", "", 100.0f); csConf.setResourceComparator(DominantResourceCalculator.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 61d9a2ddd0aec..1ed832e6b6d26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -1343,7 +1343,7 @@ public void testContainerRejectionWhenAskBeyondDynamicMax() .getConfigurationWithMultipleQueues(conf); newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class, ResourceCalculator.class); - newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a") + newConf.set(QueuePrefixes.getQueuePrefix("root.a") + MAXIMUM_ALLOCATION_MB, "4096"); MockRM rm1 = new MockRM(newConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index cf5ae842fd0a5..f9696ee41e0b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -45,6 +45,9 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; + public class TestQueueParsing { private static final Logger LOG = @@ -95,7 +98,7 @@ private void setupQueueConfigurationWithSpacesShouldBeTrimmed( CapacitySchedulerConfiguration conf) { // Define top-level queues conf.set( - CapacitySchedulerConfiguration + QueuePrefixes .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); @@ -115,7 +118,7 @@ private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed( CapacitySchedulerConfiguration conf) { // Define top-level queues conf.set( - CapacitySchedulerConfiguration + QueuePrefixes .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); @@ -131,7 +134,7 @@ private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed( conf.setMaximumCapacity(C, 70); // sub queues for A - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A) + conf.set(QueuePrefixes.getQueuePrefix(A) + CapacitySchedulerConfiguration.QUEUES, "a1, a2 "); final String A1 = CapacitySchedulerConfiguration.ROOT + ".a.a1"; @@ -1155,16 +1158,16 @@ public void testQueueCapacityWithWeight() throws Exception { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a" }); - conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100); - conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100); + conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "x", 100); + conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "y", 100); + conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "z", 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - conf.setNonLabeledQueueWeight(A, 100); + conf.setNonLabeledQueueWeight(A_QUEUE_PATH, 100); conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z")); - conf.setLabeledQueueWeight(A, "x", 100); - conf.setLabeledQueueWeight(A, "y", 100); - conf.setLabeledQueueWeight(A, "z", 70); + conf.setLabeledQueueWeight(A_QUEUE_PATH, "x", 100); + conf.setLabeledQueueWeight(A_QUEUE_PATH, "y", 100); + conf.setLabeledQueueWeight(A_QUEUE_PATH, "z", 70); MockRM rm = null; try { rm = new MockRM(conf) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 28ca66847de00..2cac58bd8bb22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -55,6 +55,7 @@ import java.util.Map; import java.util.Set; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_QUEUE_PATH; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -311,7 +312,7 @@ public static Configuration getConfigurationWithQueueLabels(Configuration config conf.setCapacity(B1, 100); conf.setMaximumCapacity(B1, 100); conf.setCapacityByLabel(B1, "y", 100); - conf.setMaximumApplicationMasterResourcePerQueuePercent(B1, 1f); + conf.setMaximumApplicationMasterResourcePerQueuePercent(B1_QUEUE_PATH, 1f); final String C1 = C + ".c1"; conf.setQueues(C, new String[] {"c1"}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java index 1aba816abd27a..1d33165601421 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityVectorEntry; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Assert; import org.junit.Test; @@ -36,6 +37,7 @@ import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestQueueMetricsForCustomResources.GB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix; public class TestQueueCapacityConfigParser { @@ -51,6 +53,7 @@ public class TestQueueCapacityConfigParser { private static final float WEIGHT_VALUE = 6; private static final String QUEUE = "root.test"; + private static final QueuePath QUEUE_PATH = new QueuePath("root.test"); private static final String ABSOLUTE_RESOURCE = String.format( ALL_RESOURCE_TEMPLATE, MEMORY_ABSOLUTE, VCORE_ABSOLUTE, GPU_ABSOLUTE); @@ -100,7 +103,7 @@ public void testPercentageCapacityConfig() { @Test public void testWeightCapacityConfig() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.setNonLabeledQueueWeight(QUEUE, WEIGHT_VALUE); + conf.setNonLabeledQueueWeight(QUEUE_PATH, WEIGHT_VALUE); QueueCapacityVector weightCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); @@ -117,8 +120,7 @@ public void testWeightCapacityConfig() { @Test public void testAbsoluteCapacityVectorConfig() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + - CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE); + conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE); conf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_TYPES); ResourceUtils.resetResourceTypes(conf); @@ -139,8 +141,8 @@ public void testAbsoluteCapacityVectorConfig() { Assert.assertEquals(GPU_ABSOLUTE, absoluteCapacityVector.getResource(GPU_URI) .getResourceValue(), EPSILON); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + - CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE_MEMORY_VCORE); + conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, + ABSOLUTE_RESOURCE_MEMORY_VCORE); QueueCapacityVector withoutGpuVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); Assert.assertEquals(3, withoutGpuVector.getResourceCount()); @@ -150,8 +152,7 @@ public void testAbsoluteCapacityVectorConfig() { @Test public void testMixedCapacityConfig() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) - + CapacitySchedulerConfiguration.CAPACITY, MIXED_RESOURCE); + conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY,MIXED_RESOURCE); conf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_TYPES); ResourceUtils.resetResourceTypes(conf); @@ -174,8 +175,8 @@ public void testMixedCapacityConfig() { mixedCapacityVector.getResource(GPU_URI).getResourceValue(), EPSILON); // Test undefined capacity type default value - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) - + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE_MEMORY_VCORE); + conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, + ABSOLUTE_RESOURCE_MEMORY_VCORE); QueueCapacityVector mixedCapacityVectorWithGpuUndefined = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); @@ -190,7 +191,7 @@ public void testMixedCapacityConfig() { public void testInvalidCapacityConfigs() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, NONEXISTINGSUFFIX); QueueCapacityVector capacityVectorWithInvalidSuffix = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); @@ -198,7 +199,7 @@ public void testInvalidCapacityConfigs() { Lists.newArrayList(capacityVectorWithInvalidSuffix.iterator()); Assert.assertEquals(0, entriesWithInvalidSuffix.size()); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, INVALID_CAPACITY_FORMAT); QueueCapacityVector invalidDelimiterCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); @@ -206,7 +207,7 @@ public void testInvalidCapacityConfigs() { Lists.newArrayList(invalidDelimiterCapacityVector.iterator()); Assert.assertEquals(0, invalidDelimiterEntries.size()); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, INVALID_CAPACITY_BRACKET); QueueCapacityVector invalidCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); @@ -214,24 +215,21 @@ public void testInvalidCapacityConfigs() { Lists.newArrayList(invalidCapacityVector.iterator()); Assert.assertEquals(0, resources.size()); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) - + CapacitySchedulerConfiguration.CAPACITY, EMPTY_BRACKET); + conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, EMPTY_BRACKET); QueueCapacityVector emptyBracketCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); List emptyEntries = Lists.newArrayList(emptyBracketCapacityVector.iterator()); Assert.assertEquals(0, emptyEntries.size()); - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) - + CapacitySchedulerConfiguration.CAPACITY, ""); + conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, ""); QueueCapacityVector emptyCapacity = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); List emptyResources = Lists.newArrayList(emptyCapacity.iterator()); Assert.assertEquals(emptyResources.size(), 0); - conf.unset(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) - + CapacitySchedulerConfiguration.CAPACITY); + conf.unset(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY); QueueCapacityVector nonSetCapacity = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); List nonSetResources = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java index ac5caedc9db0c..28c41c16e5c1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; import static org.junit.Assert.assertEquals; import java.util.*; @@ -31,6 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.junit.Assert; import org.junit.Test; @@ -163,12 +165,12 @@ public void testSizeBasedWeightNotAffectAppActivation() throws Exception { new CapacitySchedulerConfiguration(); // Define top-level queues - String queuePath = CapacitySchedulerConfiguration.ROOT + ".default"; + QueuePath queuePath = DEFAULT_QUEUE_PATH; csConf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName()); - csConf.setOrderingPolicy(queuePath, + csConf.setOrderingPolicy(queuePath.getFullPath(), CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY); - csConf.setOrderingPolicyParameter(queuePath, + csConf.setOrderingPolicyParameter(queuePath.getFullPath(), FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true"); csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index ec65237fa6e17..0c9598f25923e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; @@ -124,7 +125,7 @@ public static void setupQueueConfiguration( final String a = CapacitySchedulerConfiguration.ROOT + ".a"; config.setCapacity(a, 10.5f); config.setMaximumCapacity(a, 50); - config.setInt(CapacitySchedulerConfiguration.getQueuePrefix(a) + MAX_PARALLEL_APPLICATIONS, 42); + config.setInt(QueuePrefixes.getQueuePrefix(a) + MAX_PARALLEL_APPLICATIONS, 42); final String b = CapacitySchedulerConfiguration.ROOT + ".b"; config.setCapacity(b, 89.5f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index f2336e79d2ace..5055b3efcfa8d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.junit.Test; @@ -145,7 +146,7 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() throws Exception { Configuration config = CSConfigGenerator .createWeightConfigWithAutoQueueCreationEnabled(); - config.setInt(CapacitySchedulerConfiguration + config.setInt(QueuePrefixes .getQueuePrefix("root.autoParent1") + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + "maximum-applications", 300); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 43b5ee7127c2e..9579ce25f34f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; @@ -67,6 +68,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_QUEUE_PATH; import static org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils.toJson; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -718,7 +720,7 @@ public void testUpdateQueue() throws Exception { assertEquals(CapacitySchedulerConfiguration .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, cs.getConfiguration() - .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), + .getMaximumApplicationMasterResourcePerQueuePercent(A_QUEUE_PATH), 0.001f); response = r.path("ws").path("v1").path("cluster") @@ -731,7 +733,7 @@ public void testUpdateQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = cs.getConfiguration(); assertEquals(0.2f, newCSConf - .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), 0.001f); + .getMaximumApplicationMasterResourcePerQueuePercent(A_QUEUE_PATH), 0.001f); // Remove config. Config value should be reverted to default. updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX, @@ -750,7 +752,7 @@ public void testUpdateQueue() throws Exception { newCSConf = cs.getConfiguration(); assertEquals(CapacitySchedulerConfiguration .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, newCSConf - .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), + .getMaximumApplicationMasterResourcePerQueuePercent(A_QUEUE_PATH), 0.001f); } @@ -1021,8 +1023,8 @@ public void testNodeLabelRemovalResidualConfigsAreCleared() throws Exception { private String getConfValueForQueueAndLabelAndType(CapacityScheduler cs, QueuePath queuePath, String label, String type) { return cs.getConfiguration().get( - CapacitySchedulerConfiguration.getNodeLabelPrefix( - queuePath.getFullPath(), label) + type); + QueuePrefixes.getNodeLabelPrefix( + queuePath, label) + type); } private Object logAndReturnJson(WebResource ws, String json) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java index e37a8d83c0560..ac75abeab0aab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java @@ -59,6 +59,7 @@ import java.util.List; import java.util.function.Predicate; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_ACT_ALLOCATIONS; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_ACT_ALLOCATION_STATE; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_ACT_COUNT; @@ -140,13 +141,11 @@ private static void setupQueueConfiguration( config.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - final String queueA = CapacitySchedulerConfiguration.ROOT + ".a"; - config.setCapacity(queueA, 10.5f); - config.setMaximumCapacity(queueA, 50); + config.setCapacity(A, 10.5f); + config.setMaximumCapacity(A, 50); - final String queueB = CapacitySchedulerConfiguration.ROOT + ".b"; - config.setCapacity(queueB, 89.5f); - config.setMaximumApplicationMasterResourcePerQueuePercent(queueB, 100); + config.setCapacity(B, 89.5f); + config.setMaximumApplicationMasterResourcePerQueuePercent(B_QUEUE_PATH, 100); } @Before From 3f6bfeeb2740dd287c675ecad952e3757e82c5ee Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Mon, 30 Jan 2023 09:07:01 +0100 Subject: [PATCH 02/21] YARN-11041. Replace all occurences of queuePath with the new QueuePath class - followup Change-Id: Ifdaae6f0b3b5be0efbd841988973f3f91a67f22e --- .../CapacitySchedulerConfiguration.java | 4 +- .../resourcemanager/TestAppManager.java | 14 +- .../ReservationSystemTestUtil.java | 5 + .../TestCapacitySchedulerPlanFollower.java | 5 +- .../CapacitySchedulerQueueHelpers.java | 125 ++++++++++-------- .../capacity/TestCapacityScheduler.java | 34 ++--- ...CapacitySchedulerAutoCreatedQueueBase.java | 56 +++++--- .../capacity/TestCapacitySchedulerPerf.java | 7 +- .../TestCapacitySchedulerWeightMode.java | 36 ++--- ...pacitySchedulerWithMultiResourceTypes.java | 14 +- .../capacity/TestContainerAllocation.java | 17 +-- .../scheduler/capacity/TestLeafQueue.java | 8 +- 12 files changed, 190 insertions(+), 135 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index a050825ca115a..e34795c001165 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -2532,9 +2532,9 @@ public void setAutoCreatedLeafQueueTemplateCapacityByLabel(String queuePath, @Private @VisibleForTesting - public void setAutoCreatedLeafQueueConfigMaxCapacity(String queuePath, + public void setAutoCreatedLeafQueueConfigMaxCapacity(QueuePath queuePath, float val) { - String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( + QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix( queuePath); setMaximumCapacity(leafQueueConfPrefix, val); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index b74fdd2e65108..bdfd58ef1407b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -152,6 +152,8 @@ public class TestAppManager extends AppManagerTestBase{ private static final String USER_ID_PREFIX = "userid="; private static final String ROOT_PARENT = PREFIX + "root.parent."; + private static final QueuePath TEST_QUEUE_PATH = new QueuePath("root.test"); + private static final QueuePath MANAGED_PARENT_QUEUE_PATH = new QueuePath("root.managedparent"); public synchronized RMAppEventType getAppEventType() { return appEventType; @@ -326,8 +328,8 @@ public void testQueueSubmitWithACLsEnabledWithQueueMapping() csConf.setCapacity(DEFAULT_QUEUE_PATH, 50.0f); csConf.setMaximumCapacity(DEFAULT_QUEUE_PATH, 100.0f); - csConf.setCapacity("root.test", 50.0f); - csConf.setMaximumCapacity("root.test", 100.0f); + csConf.setCapacity(TEST_QUEUE_PATH, 50.0f); + csConf.setMaximumCapacity(TEST_QUEUE_PATH, 100.0f); csConf.set(PREFIX + "root.acl_submit_applications", " "); csConf.set(PREFIX + "root.acl_administer_queue", " "); @@ -365,11 +367,11 @@ public void testQueueSubmitWithACLsEnabledWithQueueMappingForLegacyAutoCreatedQu conf, false); csConf.set(PREFIX + "root.queues", "default,managedparent"); - csConf.setCapacity("root.default", 50.0f); - csConf.setMaximumCapacity("root.default", 100.0f); + csConf.setCapacity(DEFAULT_QUEUE_PATH, 50.0f); + csConf.setMaximumCapacity(DEFAULT_QUEUE_PATH, 100.0f); - csConf.setCapacity("root.managedparent", 50.0f); - csConf.setMaximumCapacity("root.managedparent", 100.0f); + csConf.setCapacity(MANAGED_PARENT_QUEUE_PATH, 50.0f); + csConf.setMaximumCapacity(MANAGED_PARENT_QUEUE_PATH, 100.0f); csConf.set(PREFIX + "root.acl_submit_applications", " "); csConf.set(PREFIX + "root.acl_administer_queue", " "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index bf5fb8ca1a4d5..cb28fc719fa8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; @@ -333,6 +334,10 @@ public static String getFullReservationQueueName() { + CapacitySchedulerConfiguration.DOT + RESERVATION_Q_SHORT; } + public static QueuePath getFullReservationQueuePath() { + return new QueuePath(getFullReservationQueueName()); + } + public static String getReservationQueueName() { return reservationQ; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java index b4ad8be8587f9..c972c6ce7941f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -128,9 +129,11 @@ private void setupPlanFollower() throws Exception { String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName(); + QueuePath reservationQueuePath = + ReservationSystemTestUtil.getFullReservationQueuePath(); CapacitySchedulerConfiguration csConf = cs.getConfiguration(); csConf.setReservationWindow(reservationQ, 20L); - csConf.setMaximumCapacity(reservationQ, 40); + csConf.setMaximumCapacity(reservationQueuePath, 40); csConf.setAverageCapacity(reservationQ, 20); policy.init(reservationQ, csConf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java index 6c17345680d2e..1f1ae5c752186 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java @@ -30,21 +30,30 @@ public final class CapacitySchedulerQueueHelpers { public static final String DEFAULT = CapacitySchedulerConfiguration.ROOT + ".default"; public static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; public static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + public static final String C = CapacitySchedulerConfiguration.ROOT + ".c"; public static final String A_CHILD = A + ".a"; public static final String A1 = A + ".a1"; public static final String A2 = A + ".a2"; + public static final String A3 = A + ".a3"; public static final String B1 = B + ".b1"; public static final String B2 = B + ".b2"; public static final String B3 = B + ".b3"; + public static final String C1 = C + ".c1"; + public static final String C2 = C + ".c2"; public static final QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); public static final QueuePath DEFAULT_QUEUE_PATH = new QueuePath(DEFAULT); public static final QueuePath A_QUEUE_PATH = new QueuePath(A); - public static final QueuePath B_QUEUE_PATH = new QueuePath(B); + public static final QueuePath A_CHILD_QUEUE_PATH = new QueuePath(A_CHILD); public static final QueuePath A1_QUEUE_PATH = new QueuePath(A1); public static final QueuePath A2_QUEUE_PATH = new QueuePath(A2); + public static final QueuePath A3_QUEUE_PATH = new QueuePath(A3); + public static final QueuePath B_QUEUE_PATH = new QueuePath(B); public static final QueuePath B1_QUEUE_PATH = new QueuePath(B1); public static final QueuePath B2_QUEUE_PATH = new QueuePath(B2); public static final QueuePath B3_QUEUE_PATH = new QueuePath(B3); + public static final QueuePath C_QUEUE_PATH = new QueuePath(C); + public static final QueuePath C1_QUEUE_PATH = new QueuePath(C1); + public static final QueuePath C2_QUEUE_PATH = new QueuePath(C2); public static final float A_CAPACITY = 10.5f; public static final float B_CAPACITY = 89.5f; public static final String P1 = CapacitySchedulerConfiguration.ROOT + ".p1"; @@ -53,6 +62,12 @@ public final class CapacitySchedulerQueueHelpers { public static final String X2 = P1 + ".x2"; public static final String Y1 = P2 + ".y1"; public static final String Y2 = P2 + ".y2"; + public static final QueuePath P1_QUEUE_PATH = new QueuePath(P1); + public static final QueuePath P2_QUEUE_PATH = new QueuePath(P2); + public static final QueuePath X1_QUEUE_PATH = new QueuePath(X1); + public static final QueuePath X2_QUEUE_PATH = new QueuePath(X2); + public static final QueuePath Y1_QUEUE_PATH = new QueuePath(Y1); + public static final QueuePath Y2_QUEUE_PATH = new QueuePath(Y2); public static final float A1_CAPACITY = 30; public static final float A2_CAPACITY = 70; public static final float B1_CAPACITY = 79.2f; @@ -79,22 +94,22 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"}); - conf.setCapacity(A, A_CAPACITY); - conf.setCapacity(B, B_CAPACITY); + conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); + conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1, A1_CAPACITY); + conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2, A2_CAPACITY); + conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); conf.setQueues(B, new String[]{"b1", "b2", "b3"}); - conf.setCapacity(B1, B1_CAPACITY); + conf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); - conf.setCapacity(B2, B2_CAPACITY); + conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3, B3_CAPACITY); + conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); return conf; @@ -106,24 +121,24 @@ public static CapacitySchedulerConfiguration setupAdditionalQueues( // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"}); - conf.setCapacity(A, A_CAPACITY); - conf.setCapacity(B, B_CAPACITY); + conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); + conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2", "a3"}); - conf.setCapacity(A1, 30.0f); + conf.setCapacity(A1_QUEUE_PATH, 30.0f); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2, 30.0f); + conf.setCapacity(A2_QUEUE_PATH, 30.0f); conf.setUserLimitFactor(A2, 100.0f); - conf.setCapacity("root.a.a3", 40.0f); + conf.setCapacity(A3_QUEUE_PATH, 40.0f); conf.setUserLimitFactor("root.a.a3", 100.0f); conf.setQueues(B, new String[]{"b1", "b2", "b3"}); - conf.setCapacity(B1, B1_CAPACITY); + conf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); - conf.setCapacity(B2, B2_CAPACITY); + conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3, B3_CAPACITY); + conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); return conf; @@ -145,14 +160,14 @@ public static CapacitySchedulerConfiguration setupQueueConfAmbiguousQueue( conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"}); - conf.setCapacity(A, A_CAPACITY); - conf.setCapacity(B, B_CAPACITY); + conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); + conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a", "a1"}); - conf.setCapacity(A_CHILD, A1_CAPACITY); + conf.setCapacity(A_CHILD_QUEUE_PATH, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A1, A2_CAPACITY); + conf.setCapacity(A1_QUEUE_PATH, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); return conf; @@ -174,14 +189,14 @@ public static CapacitySchedulerConfiguration setupQueueConfWithoutChildrenOfB( conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"}); - conf.setCapacity(A, A_CAPACITY); - conf.setCapacity(B, B_CAPACITY); + conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); + conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1, A1_CAPACITY); + conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2, A2_CAPACITY); + conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); return conf; @@ -203,20 +218,20 @@ public static CapacitySchedulerConfiguration setupQueueConfigurationWithoutB1( conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"}); - conf.setCapacity(A, A_CAPACITY); - conf.setCapacity(B, B_CAPACITY); + conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); + conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1, A1_CAPACITY); + conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2, A2_CAPACITY); + conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); conf.setQueues(B, new String[]{"b2", "b3"}); - conf.setCapacity(B2, B2_CAPACITY + B1_CAPACITY); //as B1 is deleted + conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY + B1_CAPACITY); //as B1 is deleted conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3, B3_CAPACITY); + conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); return conf; @@ -240,28 +255,28 @@ public static CapacitySchedulerConfiguration setupQueueConfigurationWithB1AsPare conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"}); - conf.setCapacity(A, A_CAPACITY); - conf.setCapacity(B, B_CAPACITY); + conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); + conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1, A1_CAPACITY); + conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2, A2_CAPACITY); + conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); conf.setQueues(B, new String[]{"b1", "b2", "b3"}); - conf.setCapacity(B1, B1_CAPACITY); + conf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); - conf.setCapacity(B2, B2_CAPACITY); + conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3, B3_CAPACITY); + conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); // Set childQueue for B1 conf.setQueues(B1, new String[]{"b11"}); final String b11 = B1 + ".b11"; - conf.setCapacity(b11, 100.0f); + conf.setCapacity(new QueuePath(b11), 100.0f); conf.setUserLimitFactor(b11, 100.0f); return conf; @@ -277,13 +292,13 @@ public static CapacitySchedulerConfiguration setupQueueConfigurationWithoutB( // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a"}); - conf.setCapacity(A, A_CAPACITY + B_CAPACITY); + conf.setCapacity(A_QUEUE_PATH, A_CAPACITY + B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1, A1_CAPACITY); + conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2, A2_CAPACITY); + conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); return conf; @@ -296,12 +311,12 @@ public static CapacitySchedulerConfiguration setupBlockedQueueConfiguration( conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"}); - conf.setCapacity(A, 80f); - conf.setCapacity(B, 20f); + conf.setCapacity(A_QUEUE_PATH, 80f); + conf.setCapacity(B_QUEUE_PATH, 20f); conf.setUserLimitFactor(A, 100); conf.setUserLimitFactor(B, 100); - conf.setMaximumCapacity(A, 100); - conf.setMaximumCapacity(B, 100); + conf.setMaximumCapacity(A_QUEUE_PATH, 100); + conf.setMaximumCapacity(B_QUEUE_PATH, 100); return conf; } @@ -312,23 +327,23 @@ public static CapacitySchedulerConfiguration setupOtherBlockedQueueConfiguration conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"p1", "p2"}); - conf.setCapacity(P1, 50f); - conf.setMaximumCapacity(P1, 50f); - conf.setCapacity(P2, 50f); - conf.setMaximumCapacity(P2, 100f); + conf.setCapacity(P1_QUEUE_PATH, 50f); + conf.setMaximumCapacity(P1_QUEUE_PATH, 50f); + conf.setCapacity(P2_QUEUE_PATH, 50f); + conf.setMaximumCapacity(P2_QUEUE_PATH, 100f); // Define 2nd-level queues conf.setQueues(P1, new String[]{"x1", "x2"}); - conf.setCapacity(X1, 80f); - conf.setMaximumCapacity(X1, 100f); + conf.setCapacity(X1_QUEUE_PATH, 80f); + conf.setMaximumCapacity(X1_QUEUE_PATH, 100f); conf.setUserLimitFactor(X1, 2f); - conf.setCapacity(X2, 20f); - conf.setMaximumCapacity(X2, 100f); + conf.setCapacity(X2_QUEUE_PATH, 20f); + conf.setMaximumCapacity(X2_QUEUE_PATH, 100f); conf.setUserLimitFactor(X2, 2f); conf.setQueues(P2, new String[]{"y1", "y2"}); - conf.setCapacity(Y1, 80f); + conf.setCapacity(Y1_QUEUE_PATH, 80f); conf.setUserLimitFactor(Y1, 2f); - conf.setCapacity(Y2, 20f); + conf.setCapacity(Y2_QUEUE_PATH, 20f); conf.setUserLimitFactor(Y2, 2f); return conf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 6a712027dd68d..762ca2af0e1ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -576,9 +576,9 @@ public void testMaximumCapacitySetup() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE, conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta); - conf.setMaximumCapacity(A, 50.0f); + conf.setMaximumCapacity(A_QUEUE_PATH, 50.0f); assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta); - conf.setMaximumCapacity(A, -1); + conf.setMaximumCapacity(A_QUEUE_PATH, -1); assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE, conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta); } @@ -627,7 +627,7 @@ public void testParseQueueWithAbsoluteResource() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.setQueues("root", new String[] {childQueue}); - conf.setCapacity("root." + childQueue, "[memory=20480,vcores=200]"); + conf.setCapacity(new QueuePath("root." + childQueue), "[memory=20480,vcores=200]"); conf.setAccessibleNodeLabels("root." + childQueue, Sets.newHashSet(labelName)); conf.setCapacityByLabel("root", labelName, "[memory=10240,vcores=100]"); @@ -1529,17 +1529,17 @@ public void testParentQueueMaxCapsAreRespected() throws Exception { */ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - csConf.setCapacity(A, 50); - csConf.setMaximumCapacity(A, 50); - csConf.setCapacity(B, 50); + csConf.setCapacity(A_QUEUE_PATH, 50); + csConf.setMaximumCapacity(A_QUEUE_PATH, 50); + csConf.setCapacity(B_QUEUE_PATH, 50); // Define 2nd-level queues csConf.setQueues(A, new String[] {"a1", "a2"}); - csConf.setCapacity(A1, 50); + csConf.setCapacity(A1_QUEUE_PATH, 50); csConf.setUserLimitFactor(A1, 100.0f); - csConf.setCapacity(A2, 50); + csConf.setCapacity(A2_QUEUE_PATH, 50); csConf.setUserLimitFactor(A2, 100.0f); - csConf.setCapacity(B1, B1_CAPACITY); + csConf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); csConf.setUserLimitFactor(B1, 100.0f); YarnConfiguration conf = new YarnConfiguration(csConf); @@ -2528,7 +2528,7 @@ private CapacityScheduler setUpCSQueue(long maxLifetime, new CapacitySchedulerConfiguration(); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"default"}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".default", 100); + csConf.setCapacity(DEFAULT_QUEUE_PATH, 100); csConf.setMaximumLifetimePerQueue( CapacitySchedulerConfiguration.ROOT + ".default", maxLifetime); csConf.setDefaultLifetimePerQueue( @@ -2867,9 +2867,9 @@ ImmutableMap. builder() // Add child queue to a, and reinitialize. Metrics should be updated csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 29.5f); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.5f); - csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", + csConf.setCapacity(A2_QUEUE_PATH, 29.5f); + csConf.setCapacity(A3_QUEUE_PATH, 40.5f); + csConf.setMaximumCapacity(A3_QUEUE_PATH, 50.0f); cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, @@ -2916,11 +2916,11 @@ public void testReservedContainerLeakWhenMoveApplication() throws Exception { = new CapacitySchedulerConfiguration(); csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - csConf.setCapacity("root.a", 50); - csConf.setMaximumCapacity("root.a", 100); + csConf.setCapacity(A_QUEUE_PATH, 50); + csConf.setMaximumCapacity(A_QUEUE_PATH, 100); csConf.setUserLimitFactor("root.a", 100); - csConf.setCapacity("root.b", 50); - csConf.setMaximumCapacity("root.b", 100); + csConf.setCapacity(B_QUEUE_PATH, 50); + csConf.setMaximumCapacity(B_QUEUE_PATH, 100); csConf.setUserLimitFactor("root.b", 100); YarnConfiguration conf=new YarnConfiguration(csConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index a883182a283e4..61fcd68f703ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -116,6 +116,12 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { public static final String C = CapacitySchedulerConfiguration.ROOT + ".c"; public static final String D = CapacitySchedulerConfiguration.ROOT + ".d"; public static final String E = CapacitySchedulerConfiguration.ROOT + ".e"; + + public static final QueuePath A_QUEUE_PATH = new QueuePath(A); + public static final QueuePath B_QUEUE_PATH = new QueuePath(B); + public static final QueuePath C_QUEUE_PATH = new QueuePath(C); + public static final QueuePath D_QUEUE_PATH = new QueuePath(D); + public static final QueuePath E_QUEUE_PATH = new QueuePath(E); public static final String ESUBGROUP1 = CapacitySchedulerConfiguration.ROOT + ".esubgroup1"; public static final String FGROUP = @@ -128,6 +134,18 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { public static final String B4 = B + ".b4subgroup1"; public static final String ESUBGROUP1_A = ESUBGROUP1 + ".e"; public static final String FGROUP_F = FGROUP + ".f"; + + public static final QueuePath A1_QUEUE_PATH = new QueuePath(A1); + public static final QueuePath A2_QUEUE_PATH = new QueuePath(A2); + public static final QueuePath B1_QUEUE_PATH = new QueuePath(B1); + public static final QueuePath B2_QUEUE_PATH = new QueuePath(B2); + public static final QueuePath B3_QUEUE_PATH = new QueuePath(B3); + public static final QueuePath B4_QUEUE_PATH = new QueuePath(B4); + public static final QueuePath E_GROUP_QUEUE_PATH = new QueuePath(ESUBGROUP1); + public static final QueuePath F_GROUP_QUEUE_PATH = new QueuePath(FGROUP); + public static final QueuePath E_SG_QUEUE_PATH = new QueuePath(ESUBGROUP1_A); + public static final QueuePath F_SG_QUEUE_PATH = new QueuePath(FGROUP_F); + public static final float A_CAPACITY = 20f; public static final float B_CAPACITY = 20f; public static final float C_CAPACITY = 20f; @@ -371,35 +389,35 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( new String[] {"a", "b", "c", "d", "esubgroup1", "esubgroup2", "fgroup", "a1group", "ggroup", "g"}); - conf.setCapacity(A, A_CAPACITY); - conf.setCapacity(B, B_CAPACITY); - conf.setCapacity(C, C_CAPACITY); - conf.setCapacity(D, D_CAPACITY); - conf.setCapacity(ESUBGROUP1, ESUBGROUP1_CAPACITY); - conf.setCapacity(FGROUP, FGROUP_CAPACITY); + conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); + conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); + conf.setCapacity(C_QUEUE_PATH, C_CAPACITY); + conf.setCapacity(D_QUEUE_PATH, D_CAPACITY); + conf.setCapacity(E_GROUP_QUEUE_PATH, ESUBGROUP1_CAPACITY); + conf.setCapacity(F_GROUP_QUEUE_PATH, FGROUP_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[] { "a1", "a2" }); - conf.setCapacity(A1, A1_CAPACITY); + conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2, A2_CAPACITY); + conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); conf.setQueues(B, new String[] { "b1", "b2", "b3", "b4subgroup1" }); - conf.setCapacity(B1, B1_CAPACITY); + conf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); - conf.setCapacity(B2, B2_CAPACITY); + conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3, B3_CAPACITY); + conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); - conf.setCapacity(B4, B4_CAPACITY); + conf.setCapacity(B4_QUEUE_PATH, B4_CAPACITY); conf.setUserLimitFactor(B4, 100.0f); conf.setQueues(ESUBGROUP1, new String[] {"e"}); - conf.setCapacity(ESUBGROUP1_A, 100f); + conf.setCapacity(E_SG_QUEUE_PATH, 100f); conf.setUserLimitFactor(ESUBGROUP1_A, 100.0f); conf.setQueues(FGROUP, new String[] {"f"}); - conf.setCapacity(FGROUP_F, 100f); + conf.setCapacity(F_SG_QUEUE_PATH, 100f); conf.setUserLimitFactor(FGROUP_F, 100.0f); conf.setUserLimitFactor(C, 1.0f); @@ -407,7 +425,7 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( //Setup leaf queue template configs conf.setAutoCreatedLeafQueueConfigCapacity(C, 50.0f); - conf.setAutoCreatedLeafQueueConfigMaxCapacity(C, 100.0f); + conf.setAutoCreatedLeafQueueConfigMaxCapacity(C_QUEUE_PATH, 100.0f); conf.setAutoCreatedLeafQueueConfigUserLimit(C, 100); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f); @@ -436,7 +454,7 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( //Setup leaf queue template configs conf.setAutoCreatedLeafQueueConfigCapacity(D, 10.0f); - conf.setAutoCreatedLeafQueueConfigMaxCapacity(D, 100.0f); + conf.setAutoCreatedLeafQueueConfigMaxCapacity(D_QUEUE_PATH, 100.0f); conf.setAutoCreatedLeafQueueConfigUserLimit(D, 3); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(D, 100); @@ -473,14 +491,14 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( // Set childQueue for root conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"c"}); - conf.setCapacity(C, 100f); + conf.setCapacity(C_QUEUE_PATH, 100f); conf.setUserLimitFactor(C, 1.0f); conf.setAutoCreateChildQueueEnabled(C, true); //Setup leaf queue template configs conf.setAutoCreatedLeafQueueConfigCapacity(C, 100f); - conf.setAutoCreatedLeafQueueConfigMaxCapacity(C, 100.0f); + conf.setAutoCreatedLeafQueueConfigMaxCapacity(C_QUEUE_PATH, 100.0f); conf.setAutoCreatedLeafQueueConfigUserLimit(C, 100); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f); @@ -783,7 +801,7 @@ protected void validateEffectiveMinResource(ResourceManager rm, if (expectedQueueEntitlements.get(label).getCapacity() > EPSILON) { if (leafQueue.getCapacityConfigType().equals(ABSOLUTE_RESOURCE)) { String templatePrefix = QueuePrefixes.getAutoCreatedQueueTemplateConfPrefix( - parentQueue.getQueuePath()); + parentQueue.getQueuePathObject()); Resource resourceTemplate = parentQueue.getLeafQueueTemplate().getLeafQueueConfigs() .getMinimumResourceRequirement(label, templatePrefix, RESOURCE_TYPES); assertEquals(resourceTemplate, leafQueue.getEffectiveCapacity(label)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java index ace82932e983f..73ed20a73665a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java @@ -62,6 +62,7 @@ import java.util.concurrent.atomic.AtomicLong; import static org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -451,7 +452,7 @@ CapacitySchedulerConfiguration createCSConfWithManyQueues(int numQueues) csconf.setResourceComparator(DominantResourceCalculator.class); csconf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT_QUEUE_PATH, 100.0f); csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f); - csconf.setCapacity("root.default", 0.0f); + csconf.setCapacity(DEFAULT_QUEUE_PATH, 0.0f); csconf.setOffSwitchPerHeartbeatLimit(numQueues); float capacity = 100.0f / numQueues; @@ -463,9 +464,9 @@ CapacitySchedulerConfiguration createCSConfWithManyQueues(int numQueues) csconf.setMaximumApplicationMasterResourcePerQueuePercent( queuePath, 100.0f); csconf.setMaximumAMResourcePercentPerPartition(queuePath.getFullPath(), "", 100.0f); - csconf.setCapacity(queuePath.getFullPath(), capacity); + csconf.setCapacity(queuePath, capacity); csconf.setUserLimitFactor(queuePath.getFullPath(), 100.0f); - csconf.setMaximumCapacity(queuePath.getFullPath(), 100.0f); + csconf.setMaximumCapacity(queuePath, 100.0f); } csconf.setQueues("root", subQueues); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java index 2441a9627822a..45a11ba44aac9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java @@ -60,6 +60,12 @@ public class TestCapacitySchedulerWeightMode { private static final String B1 = B + ".b1"; private static final String B2 = B + ".b2"; + private static final QueuePath A_QUEUE_PATH = new QueuePath(A); + private static final QueuePath B_QUEUE_PATH = new QueuePath(B); + private static final QueuePath A1_QUEUE_PATH = new QueuePath(A1); + private static final QueuePath B1_QUEUE_PATH = new QueuePath(B1); + private static final QueuePath B2_QUEUE_PATH = new QueuePath(B2); + private YarnConfiguration conf; RMNodeLabelsManager mgr; @@ -123,13 +129,13 @@ public static Configuration getCSConfWithQueueLabelsWeightOnly( conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "z", 100); conf.setLabeledQueueWeight(A_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(A, 10); + conf.setMaximumCapacity(A_QUEUE_PATH, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); conf.setLabeledQueueWeight(A_QUEUE_PATH, "x", 100); conf.setLabeledQueueWeight(A_QUEUE_PATH, "y", 50); conf.setLabeledQueueWeight(B_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 9); - conf.setMaximumCapacity(B, 100); + conf.setMaximumCapacity(B_QUEUE_PATH, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); conf.setLabeledQueueWeight(B_QUEUE_PATH, "y", 50); conf.setLabeledQueueWeight(B_QUEUE_PATH, "z", 100); @@ -137,7 +143,7 @@ public static Configuration getCSConfWithQueueLabelsWeightOnly( // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); conf.setLabeledQueueWeight(A1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 100); - conf.setMaximumCapacity(A1, 100); + conf.setMaximumCapacity(A1_QUEUE_PATH, 100); conf.setAccessibleNodeLabels(A1, toSet("x", "y")); conf.setDefaultNodeLabelExpression(A1, "x"); conf.setLabeledQueueWeight(A1_QUEUE_PATH, "x", 100); @@ -145,11 +151,11 @@ public static Configuration getCSConfWithQueueLabelsWeightOnly( conf.setQueues(B, new String[] { "b1", "b2" }); conf.setLabeledQueueWeight(B1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 50); - conf.setMaximumCapacity(B1, 50); + conf.setMaximumCapacity(B1_QUEUE_PATH, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); conf.setLabeledQueueWeight(B2_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 50); - conf.setMaximumCapacity(B2, 50); + conf.setMaximumCapacity(B2_QUEUE_PATH, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); conf.setLabeledQueueWeight(B2_QUEUE_PATH, "y", 100); conf.setLabeledQueueWeight(B2_QUEUE_PATH, "z", 100); @@ -182,13 +188,13 @@ public static Configuration getCSConfWithLabelsParentUseWeightChildUsePct( conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "z", 100); conf.setLabeledQueueWeight(A_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(A, 10); + conf.setMaximumCapacity(A_QUEUE_PATH, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); conf.setLabeledQueueWeight(A_QUEUE_PATH, "x", 100); conf.setLabeledQueueWeight(A_QUEUE_PATH, "y", 50); conf.setLabeledQueueWeight(B_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 9); - conf.setMaximumCapacity(B, 100); + conf.setMaximumCapacity(B_QUEUE_PATH, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); conf.setLabeledQueueWeight(B_QUEUE_PATH, "y", 50); conf.setLabeledQueueWeight(B_QUEUE_PATH, "z", 100); @@ -196,7 +202,7 @@ public static Configuration getCSConfWithLabelsParentUseWeightChildUsePct( // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100); - conf.setMaximumCapacity(A1, 100); + conf.setMaximumCapacity(A1_QUEUE_PATH, 100); conf.setAccessibleNodeLabels(A1, toSet("x", "y")); conf.setDefaultNodeLabelExpression(A1, "x"); conf.setCapacityByLabel(A1, "x", 100); @@ -204,11 +210,11 @@ public static Configuration getCSConfWithLabelsParentUseWeightChildUsePct( conf.setQueues(B, new String[] { "b1", "b2" }); conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50); - conf.setMaximumCapacity(B1, 50); + conf.setMaximumCapacity(B1_QUEUE_PATH, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50); - conf.setMaximumCapacity(B2, 50); + conf.setMaximumCapacity(B2_QUEUE_PATH, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); conf.setCapacityByLabel(B2, "y", 100); conf.setCapacityByLabel(B2, "z", 100); @@ -241,13 +247,13 @@ public static Configuration getCSConfWithLabelsParentUsePctChildUseWeight( conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 10); - conf.setMaximumCapacity(A, 10); + conf.setMaximumCapacity(A_QUEUE_PATH, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); conf.setCapacityByLabel(A, "x", 100); conf.setCapacityByLabel(A, "y", 50); conf.setCapacityByLabel(B, RMNodeLabelsManager.NO_LABEL, 90); - conf.setMaximumCapacity(B, 100); + conf.setMaximumCapacity(B_QUEUE_PATH, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); conf.setCapacityByLabel(B, "y", 50); conf.setCapacityByLabel(B, "z", 100); @@ -255,7 +261,7 @@ public static Configuration getCSConfWithLabelsParentUsePctChildUseWeight( // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); conf.setLabeledQueueWeight(A1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(A1, 100); + conf.setMaximumCapacity(A1_QUEUE_PATH, 100); conf.setAccessibleNodeLabels(A1, toSet("x", "y")); conf.setDefaultNodeLabelExpression(A1, "x"); conf.setLabeledQueueWeight(A1_QUEUE_PATH, "x", 1); @@ -263,11 +269,11 @@ public static Configuration getCSConfWithLabelsParentUsePctChildUseWeight( conf.setQueues(B, new String[] { "b1", "b2" }); conf.setLabeledQueueWeight(B1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(B1, 50); + conf.setMaximumCapacity(B1_QUEUE_PATH, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); conf.setLabeledQueueWeight(B2_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(B2, 50); + conf.setMaximumCapacity(B2_QUEUE_PATH, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); conf.setLabeledQueueWeight(B2_QUEUE_PATH, "y", 1); conf.setLabeledQueueWeight(B2_QUEUE_PATH, "z", 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index 78acb5e5f4232..ffa939324b97c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -81,6 +81,8 @@ public class TestCapacitySchedulerWithMultiResourceTypes { private static final String A_QUEUE = CapacitySchedulerConfiguration.ROOT + ".a"; private static final String B_QUEUE = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final QueuePath A_QUEUE_PATH = new QueuePath(A_QUEUE); + private static final QueuePath B_QUEUE_PATH = new QueuePath(B_QUEUE); private static float A_CAPACITY = 50.0f; private static float B_CAPACITY = 50.0f; @@ -287,9 +289,9 @@ public void testMaxLimitsOfQueueWithMultipleResources() throws Exception { new String[] {"a", "b"}); // Set each queue to consider 50% each. - csConf.setCapacity(A_QUEUE, A_CAPACITY); - csConf.setCapacity(B_QUEUE, B_CAPACITY); - csConf.setMaximumCapacity(A_QUEUE, 100.0f); + csConf.setCapacity(A_QUEUE_PATH, A_CAPACITY); + csConf.setCapacity(B_QUEUE_PATH, B_CAPACITY); + csConf.setMaximumCapacity(A_QUEUE_PATH, 100.0f); csConf.setUserLimitFactor(A_QUEUE, 2); YarnConfiguration conf = new YarnConfiguration(csConf); @@ -418,9 +420,9 @@ public void testConsumeAllExtendedResourcesWithSmallMinUserLimitPct() new String[] {"a", "b"}); // Set each queue to contain 50% each. - csconf.setCapacity(A_QUEUE, A_CAPACITY); - csconf.setCapacity(B_QUEUE, B_CAPACITY); - csconf.setMaximumCapacity(A_QUEUE, 100.0f); + csconf.setCapacity(A_QUEUE_PATH, A_CAPACITY); + csconf.setCapacity(B_QUEUE_PATH, B_CAPACITY); + csconf.setMaximumCapacity(A_QUEUE_PATH, 100.0f); csconf.setUserLimitFactor(A_QUEUE, 2); YarnConfiguration yarnConf = new YarnConfiguration(csconf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 1ed832e6b6d26..ebc9e4e98ff2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -73,6 +73,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAX_ASSIGN_PER_HEARTBEAT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; public class TestContainerAllocation { @@ -722,7 +723,7 @@ public void testContinuousReservationLookingWhenUsedEqualsMax() throws Exception (CapacitySchedulerConfiguration) TestUtils .getConfigurationWithMultipleQueues(conf); // Set maximum capacity of A to 10 - newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 10); + newConf.setMaximumCapacity(A_QUEUE_PATH, 10); MockRM rm1 = new MockRM(newConf); rm1.getRMContext().setNodeLabelManager(mgr); @@ -894,7 +895,7 @@ public void testQueuePriorityOrdering() throws Exception { CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY); // Set maximum capacity of A to 20 - newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 20); + newConf.setMaximumCapacity(A_QUEUE_PATH, 20); newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1); newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".b", 2); newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".a", 3); @@ -1185,7 +1186,7 @@ public void testUnreserveWhenClusterResourceHasEmptyResourceType() newConf .setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false); // Set maximum capacity of queue "a" to 50 - newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 50); + newConf.setMaximumCapacity(A_QUEUE_PATH, 50); MockRM rm1 = new MockRM(newConf); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); @@ -1274,12 +1275,12 @@ public void testAllocationCannotBeBlockedWhenFormerQueueReachedItsLimit() CapacitySchedulerConfiguration newConf = (CapacitySchedulerConfiguration) TestUtils .getConfigurationWithMultipleQueues(conf); - newConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".c", + newConf.setQueues(C, new String[] { "c1", "c2" }); - newConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c1", 10); + newConf.setCapacity(C1_QUEUE_PATH, 10); newConf - .setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c1", 10); - newConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 90); + .setMaximumCapacity(C1_QUEUE_PATH, 10); + newConf.setCapacity(C2_QUEUE_PATH, 90); newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class, ResourceCalculator.class); @@ -1343,7 +1344,7 @@ public void testContainerRejectionWhenAskBeyondDynamicMax() .getConfigurationWithMultipleQueues(conf); newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class, ResourceCalculator.class); - newConf.set(QueuePrefixes.getQueuePrefix("root.a") + newConf.set(QueuePrefixes.getQueuePrefix(A_QUEUE_PATH) + MAXIMUM_ALLOCATION_MB, "4096"); MockRM rm1 = new MockRM(newConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index eca065b148766..f77cc83dd11bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -26,6 +26,7 @@ .capacity.CapacitySchedulerConfiguration.DOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler .capacity.CapacitySchedulerConfiguration.ROOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -274,7 +275,7 @@ private void setupQueueConfiguration( // Define top-level queues conf.setQueues(ROOT, new String[] {newRoot}); - conf.setMaximumCapacity(ROOT, 100); + conf.setMaximumCapacity(ROOT_QUEUE_PATH, 100); conf.setAcl(ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); if (withNodeLabels) { @@ -284,9 +285,10 @@ private void setupQueueConfiguration( } final String Q_newRoot = ROOT + "." + newRoot; + final QueuePath qNewRootPath = new QueuePath(Q_newRoot); conf.setQueues(Q_newRoot, new String[] {A, B, C, D, E}); - conf.setCapacity(Q_newRoot, 100); - conf.setMaximumCapacity(Q_newRoot, 100); + conf.setCapacity(qNewRootPath, 100); + conf.setMaximumCapacity(qNewRootPath, 100); conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " "); if (withNodeLabels) { conf.setAccessibleNodeLabels(Q_newRoot, Collections.singleton(LABEL)); From 21ca3f02f91ea71442112f3ff7efa4d8780c73ee Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Tue, 7 Feb 2023 09:13:38 +0100 Subject: [PATCH 03/21] YARN-11041. Replace all occurences of queuePath with the new QueuePath class - followup Change-Id: I41a3be26a916b260005677ef228f69e094240d98 --- .../yarn/client/cli/TestSchedConfCLI.java | 2 +- .../AbstractAutoCreatedLeafQueue.java | 12 +- .../scheduler/capacity/AbstractCSQueue.java | 24 +- .../scheduler/capacity/AbstractLeafQueue.java | 18 +- .../capacity/CSQueuePreemptionSettings.java | 8 +- .../CapacitySchedulerConfigValidator.java | 2 +- .../CapacitySchedulerConfiguration.java | 350 +++++++++--------- .../CapacitySchedulerQueueManager.java | 35 +- .../capacity/ManagedParentQueue.java | 22 +- .../scheduler/capacity/ParentQueue.java | 10 +- .../scheduler/capacity/PlanQueue.java | 4 +- .../capacity/QueueAllocationSettings.java | 2 +- .../QueueAppLifetimeAndLimitSettings.java | 6 +- .../capacity/QueueNodeLabelsSettings.java | 6 +- .../scheduler/capacity/QueuePath.java | 18 + .../scheduler/capacity/QueueStateHelper.java | 2 +- .../scheduler/capacity/UserWeights.java | 2 +- .../conf/ConfigurationUpdateAssembler.java | 20 +- .../conf/QueueCapacityConfigParser.java | 5 +- .../converter/QueuePlacementConverter.java | 17 +- .../webapp/dao/CapacitySchedulerInfo.java | 5 +- .../dao/CapacitySchedulerQueueInfo.java | 11 +- .../webapp/dao/LeafQueueTemplateInfo.java | 3 +- .../server/resourcemanager/ACLsTestBase.java | 9 + .../ReservationACLsTestBase.java | 23 +- .../resourcemanager/TestAppManager.java | 96 ++--- .../TestApplicationMasterServiceCapacity.java | 26 +- .../TestWorkPreservingRMRestart.java | 61 +-- ...tProportionalCapacityPreemptionPolicy.java | 31 +- ...cityPreemptionPolicyForNodePartitions.java | 3 +- ...nalCapacityPreemptionPolicyIntraQueue.java | 14 +- .../mockframework/MockQueueHierarchy.java | 3 +- .../nodelabels/TestRMNodeLabelsManager.java | 8 +- .../ReservationSystemTestUtil.java | 56 ++- .../TestCapacitySchedulerPlanFollower.java | 4 +- .../rmapp/TestApplicationLifetimeMonitor.java | 59 ++- .../CapacitySchedulerQueueHelpers.java | 219 ++++++----- .../TestAbsoluteResourceConfiguration.java | 74 ++-- .../TestAbsoluteResourceWithAutoQueue.java | 45 ++- .../capacity/TestApplicationLimits.java | 106 +++--- .../TestApplicationLimitsByPartition.java | 35 +- ...stApplicationPriorityACLConfiguration.java | 40 +- .../capacity/TestApplicationPriorityACLs.java | 12 +- .../TestAutoCreatedQueueTemplate.java | 12 +- .../TestCSAllocateCustomResource.java | 11 +- .../TestCSMaxRunningAppsEnforcer.java | 14 +- .../capacity/TestCapacityScheduler.java | 84 ++--- .../TestCapacitySchedulerAmbiguousLeafs.java | 38 +- .../capacity/TestCapacitySchedulerApps.java | 7 +- ...CapacitySchedulerAutoCreatedQueueBase.java | 117 +++--- ...tySchedulerAutoCreatedQueuePreemption.java | 10 +- ...estCapacitySchedulerAutoQueueCreation.java | 19 +- .../TestCapacitySchedulerConfigValidator.java | 6 +- .../TestCapacitySchedulerConfiguration.java | 23 +- .../TestCapacitySchedulerDynamicBehavior.java | 19 +- .../TestCapacitySchedulerLazyPreemption.java | 3 +- .../TestCapacitySchedulerMultiNodes.java | 8 +- ...CapacitySchedulerNewQueueAutoCreation.java | 125 ++++--- .../TestCapacitySchedulerNodeLabelUpdate.java | 40 +- .../capacity/TestCapacitySchedulerNodes.java | 4 +- .../TestCapacitySchedulerOvercommit.java | 18 +- .../capacity/TestCapacitySchedulerPerf.java | 17 +- .../TestCapacitySchedulerQueueACLs.java | 44 +-- .../capacity/TestCapacitySchedulerQueues.java | 139 ++++--- ...stCapacitySchedulerSurgicalPreemption.java | 57 +-- .../TestCapacitySchedulerWeightMode.java | 154 ++++---- ...pacitySchedulerWithMultiResourceTypes.java | 52 ++- ...acitySchedulerWorkflowPriorityMapping.java | 22 +- .../capacity/TestChildQueueOrder.java | 11 +- .../capacity/TestContainerAllocation.java | 48 ++- .../scheduler/capacity/TestLeafQueue.java | 188 +++++----- .../TestNodeLabelContainerAllocation.java | 239 ++++++------ .../scheduler/capacity/TestParentQueue.java | 112 +++--- .../scheduler/capacity/TestQueueMappings.java | 9 +- .../scheduler/capacity/TestQueueParsing.java | 261 ++++++------- .../scheduler/capacity/TestQueueState.java | 21 +- .../capacity/TestQueueStateManager.java | 14 +- .../scheduler/capacity/TestReservations.java | 37 +- .../scheduler/capacity/TestUtils.java | 59 ++- .../TestConfigurationUpdateAssembler.java | 3 +- .../TestMutableCSConfigurationProvider.java | 12 +- .../conf/TestQueueCapacityConfigParser.java | 53 +-- .../constraint/TestPlacementProcessor.java | 13 +- .../TestQueuePlacementConverter.java | 3 +- .../scheduler/fifo/TestFifoScheduler.java | 6 +- .../policy/TestFairOrderingPolicy.java | 8 +- .../resourcemanager/webapp/TestRMWebApp.java | 38 +- .../webapp/TestRMWebServiceAppsNodelabel.java | 15 +- .../TestRMWebServicesAppsModification.java | 40 +- .../TestRMWebServicesCapacitySched.java | 101 ++--- ...WebServicesCapacitySchedDynamicConfig.java | 4 +- ...estRMWebServicesConfigurationMutation.java | 109 +++--- .../TestRMWebServicesForCSWithPartitions.java | 75 ++-- .../webapp/TestRMWebServicesReservation.java | 7 +- ...edulerActivitiesWithMultiNodesEnabled.java | 16 +- .../webapp/BaseRouterWebServicesTest.java | 4 + .../MockDefaultRequestInterceptorREST.java | 11 +- .../TestableFederationInterceptorREST.java | 12 +- 98 files changed, 2104 insertions(+), 1976 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java index 369f22c97641b..6cae066dc1e26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java @@ -181,7 +181,7 @@ public String getName() { private static void setupQueueConfiguration( CapacitySchedulerConfiguration config) { - config.setQueues(CapacitySchedulerConfiguration.ROOT, + config.setQueues(new QueuePath(CapacitySchedulerConfiguration.ROOT), new String[]{"testqueue"}); QueuePath a = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".testqueue"); config.setCapacity(a, 100f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index 5d3aaefe50c0b..e97e4aabe706e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -61,26 +61,26 @@ public void setEntitlement(QueueEntitlement entitlement) } @Override - protected Resource getMinimumAbsoluteResource(String queuePath, + protected Resource getMinimumAbsoluteResource(QueuePath queuePath, String label) { return super.getMinimumAbsoluteResource(QueuePrefixes - .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), + .getAutoCreatedQueueObjectTemplateConfPrefix(this.getParent().getQueuePathObject()), label); } @Override - protected Resource getMaximumAbsoluteResource(String queuePath, + protected Resource getMaximumAbsoluteResource(QueuePath queuePath, String label) { return super.getMaximumAbsoluteResource(QueuePrefixes - .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), + .getAutoCreatedQueueObjectTemplateConfPrefix(this.getParent().getQueuePathObject()), label); } @Override - protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, + protected boolean checkConfigTypeIsAbsoluteResource(QueuePath queuePath, String label) { return super.checkConfigTypeIsAbsoluteResource(QueuePrefixes - .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()), + .getAutoCreatedQueueObjectTemplateConfPrefix(this.getParent().getQueuePathObject()), label); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index f1eabab56aa5e..9e9ffaf208349 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -340,7 +340,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws writeLock.lock(); try { CapacitySchedulerConfiguration configuration = queueContext.getConfiguration(); - this.acls = configuration.getAcls(getQueuePath()); + this.acls = configuration.getAcls(getQueuePathObject()); if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) { setDynamicQueueProperties(); @@ -362,7 +362,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws // Setup queue's maximumAllocation respecting the global // and the queue settings - this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePath(), + this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePathObject(), parent); // Initialize the queue state based on previous state, configured state @@ -377,7 +377,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws configuration.getReservationContinueLook(); this.configuredCapacityVectors = configuration - .parseConfiguredResourceVector(queuePath.getFullPath(), + .parseConfiguredResourceVector(queuePath, this.queueNodeLabelsSettings.getConfiguredNodeLabels()); // Update metrics @@ -387,11 +387,11 @@ protected void setupQueueConfigs(Resource clusterResource) throws // Store preemption settings this.preemptionSettings = new CSQueuePreemptionSettings(this, configuration); this.priority = configuration.getQueuePriority( - getQueuePath()); + getQueuePathObject()); // Update multi-node sorting algorithm for scheduling as configured. setMultiNodeSortingPolicyName( - configuration.getMultiNodesSortingAlgorithmPolicy(getQueuePath())); + configuration.getMultiNodesSortingAlgorithmPolicy(getQueuePathObject())); // Setup application related limits this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration, @@ -439,21 +439,21 @@ private UserWeights getUserWeightsFromHierarchy() { // Insert this queue's userWeights, overriding parent's userWeights if // there is an overlap. unionInheritedWeights.addFrom( - queueContext.getConfiguration().getAllUserWeightsForQueue(getQueuePath())); + queueContext.getConfiguration().getAllUserWeightsForQueue(getQueuePathObject())); return unionInheritedWeights; } - protected Resource getMinimumAbsoluteResource(String queuePath, String label) { + protected Resource getMinimumAbsoluteResource(QueuePath queuePath, String label) { return queueContext.getConfiguration() .getMinimumResourceRequirement(label, queuePath, resourceTypes); } - protected Resource getMaximumAbsoluteResource(String queuePath, String label) { + protected Resource getMaximumAbsoluteResource(QueuePath queuePath, String label) { return queueContext.getConfiguration() .getMaximumResourceRequirement(label, queuePath, resourceTypes); } - protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, + protected boolean checkConfigTypeIsAbsoluteResource(QueuePath queuePath, String label) { return queueContext.getConfiguration().checkConfigTypeIsAbsoluteResource(label, queuePath, resourceTypes); @@ -466,7 +466,7 @@ protected void updateCapacityConfigType() { capacityConfigType, getQueuePath()); CapacityConfigType localType = checkConfigTypeIsAbsoluteResource( - getQueuePath(), label) ? CapacityConfigType.ABSOLUTE_RESOURCE + getQueuePathObject(), label) ? CapacityConfigType.ABSOLUTE_RESOURCE : CapacityConfigType.PERCENTAGE; if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { @@ -486,8 +486,8 @@ protected void updateCapacityConfigType() { */ protected void updateConfigurableResourceLimits(Resource clusterResource) { for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { - final Resource minResource = getMinimumAbsoluteResource(getQueuePath(), label); - Resource maxResource = getMaximumAbsoluteResource(getQueuePath(), label); + final Resource minResource = getMinimumAbsoluteResource(getQueuePathObject(), label); + Resource maxResource = getMaximumAbsoluteResource(getQueuePathObject(), label); if (parent != null) { final Resource parentMax = parent.getQueueResourceQuotas() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java index fcf102e35ce54..edd02dc6c2022 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java @@ -188,10 +188,10 @@ protected void setupQueueConfigs(Resource clusterResource) throws setQueueResourceLimitsInfo(clusterResource); setOrderingPolicy( - configuration.getAppOrderingPolicy(getQueuePath())); + configuration.getAppOrderingPolicy(getQueuePathObject())); - usersManager.setUserLimit(configuration.getUserLimit(getQueuePath())); - usersManager.setUserLimitFactor(configuration.getUserLimitFactor(getQueuePath())); + usersManager.setUserLimit(configuration.getUserLimit(getQueuePathObject())); + usersManager.setUserLimitFactor(configuration.getUserLimitFactor(getQueuePathObject())); maxAMResourcePerQueuePercent = configuration.getMaximumApplicationMasterResourcePerQueuePercent( @@ -206,7 +206,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws } } - priorityAcls = configuration.getPriorityAcls(getQueuePath(), + priorityAcls = configuration.getPriorityAcls(getQueuePathObject(), configuration.getClusterLevelApplicationMaxPriority()); Set accessibleNodeLabels = this.queueNodeLabelsSettings.getAccessibleNodeLabels(); @@ -249,10 +249,10 @@ protected void setupQueueConfigs(Resource clusterResource) throws } defaultAppPriorityPerQueue = Priority.newInstance( - configuration.getDefaultApplicationPriorityConfPerQueue(getQueuePath())); + configuration.getDefaultApplicationPriorityConfPerQueue(getQueuePathObject())); // Validate leaf queue's user's weights. - float queueUserLimit = Math.min(100.0f, configuration.getUserLimit(getQueuePath())); + float queueUserLimit = Math.min(100.0f, configuration.getUserLimit(getQueuePathObject())); getUserWeights().validateForLeafQueue(queueUserLimit, getQueuePath()); usersManager.updateUserWeights(); @@ -1694,7 +1694,7 @@ protected boolean canAssignToUser(Resource clusterResource, @Override protected void setDynamicQueueProperties() { // set to -1, to disable it - queueContext.getConfiguration().setUserLimitFactor(getQueuePath(), -1); + queueContext.getConfiguration().setUserLimitFactor(getQueuePathObject(), -1); // Set Max AM percentage to a higher value queueContext.getConfiguration().setMaximumApplicationMasterResourcePerQueuePercent( getQueuePathObject(), 1f); @@ -1707,7 +1707,7 @@ protected void setDynamicQueueACLProperties() { if (parent instanceof AbstractManagedParentQueue) { acls.putAll(queueContext.getConfiguration().getACLsForLegacyAutoCreatedLeafQueue( - parent.getQueuePath())); + parent.getQueuePathObject())); } else if (parent instanceof ParentQueue) { acls.putAll(getACLsForFlexibleAutoCreatedLeafQueue( ((ParentQueue) parent).getAutoCreatedQueueTemplate())); @@ -2473,6 +2473,6 @@ List getCopyOfNonRunnableAppSchedulables() { public boolean isEligibleForAutoDeletion() { return isDynamicQueue() && getNumApplications() == 0 && queueContext.getConfiguration(). - isAutoExpiredDeletionEnabled(this.getQueuePath()); + isAutoExpiredDeletionEnabled(this.getQueuePathObject()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java index dc254747dfec5..d429b208233b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java @@ -58,14 +58,14 @@ private boolean isQueueHierarchyPreemptionDisabled(CSQueue q, // on, then q does not have preemption disabled (default=false, below) // unless the preemption_disabled property is explicitly set. if (parentQ == null) { - return configuration.getPreemptionDisabled(q.getQueuePath(), false); + return configuration.getPreemptionDisabled(q.getQueuePathObject(), false); } // If this is not the root queue, inherit the default value for the // preemption_disabled property from the parent. Preemptability will be // inherited from the parent's hierarchy unless explicitly overridden at // this level. - return configuration.getPreemptionDisabled(q.getQueuePath(), + return configuration.getPreemptionDisabled(q.getQueuePathObject(), parentQ.getPreemptionDisabled()); } @@ -96,13 +96,13 @@ private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q, CSQueue parentQ = q.getParent(); if (parentQ == null) { return configuration - .getIntraQueuePreemptionDisabled(q.getQueuePath(), false); + .getIntraQueuePreemptionDisabled(q.getQueuePathObject(), false); } // At this point, the master preemption switch is enabled down to this // queue's level. Determine whether intra-queue preemption is enabled // down to this queue's level and return that value. - return configuration.getIntraQueuePreemptionDisabled(q.getQueuePath(), + return configuration.getIntraQueuePreemptionDisabled(q.getQueuePathObject(), parentQ.getIntraQueuePreemptionDisabledInHierarchy()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index 9748679354e88..30d50e6ebaf9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -132,7 +132,7 @@ public static void validateQueueHierarchy( final String queuePath = oldQueue.getQueuePath(); final String configPrefix = QueuePrefixes.getQueuePrefix( - oldQueue.getQueuePath()); + oldQueue.getQueuePathObject()); final QueueState newQueueState = createQueueState(newConf.get(configPrefix + "state"), queuePath); final CSQueue newQueue = newQueues.get(queuePath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index e34795c001165..8f7db5c7979a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -22,6 +22,7 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueCapacityConfigParser; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.MappingRuleCreator; import org.slf4j.Logger; @@ -427,13 +428,13 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur private ConfigurationProperties configurationProperties; - public int getMaximumAutoCreatedQueueDepth(QueuePath queuePath) { - return getInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH, + public int getMaximumAutoCreatedQueueDepth(QueuePath queue) { + return getInt(getQueuePrefix(queue) + MAXIMUM_QUEUE_DEPTH, getInt(PREFIX + MAXIMUM_QUEUE_DEPTH, DEFAULT_MAXIMUM_QUEUE_DEPTH)); } - public void setMaximumAutoCreatedQueueDepth(QueuePath queuePath, int value) { - setInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH, value); + public void setMaximumAutoCreatedQueueDepth(QueuePath queue, int value) { + setInt(getQueuePrefix(queue) + MAXIMUM_QUEUE_DEPTH, value); } public void setMaximumAutoCreatedQueueDepth(int value) { @@ -487,73 +488,73 @@ public float getMaximumApplicationMasterResourcePercent() { /** * Get the maximum applications per queue setting. - * @param queuePath path of the queue + * @param queue path of the queue * @return setting specified or -1 if not set */ - public int getMaximumApplicationsPerQueue(QueuePath queuePath) { + public int getMaximumApplicationsPerQueue(QueuePath queue) { int maxApplicationsPerQueue = - getInt(getQueuePrefix(queuePath) + MAXIMUM_APPLICATIONS_SUFFIX, + getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX, (int)UNDEFINED); return maxApplicationsPerQueue; } @VisibleForTesting - public void setMaximumApplicationsPerQueue(QueuePath queuePath, + public void setMaximumApplicationsPerQueue(QueuePath queue, int numMaxApps) { - setInt(getQueuePrefix(queuePath) + MAXIMUM_APPLICATIONS_SUFFIX, + setInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX, numMaxApps); } /** * Get the maximum am resource percent per queue setting. - * @param queuePath path of the queue + * @param queue path of the queue * @return per queue setting or defaults to the global am-resource-percent * setting if per queue setting not present */ - public float getMaximumApplicationMasterResourcePerQueuePercent(QueuePath queuePath) { - return getFloat(getQueuePrefix(queuePath) + MAXIMUM_AM_RESOURCE_SUFFIX, + public float getMaximumApplicationMasterResourcePerQueuePercent(QueuePath queue) { + return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, getMaximumApplicationMasterResourcePercent()); } - public void setMaximumApplicationMasterResourcePerQueuePercent(QueuePath queuePath, + public void setMaximumApplicationMasterResourcePerQueuePercent(QueuePath queue, float percent) { - setFloat(getQueuePrefix(queuePath) + MAXIMUM_AM_RESOURCE_SUFFIX, percent); + setFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, percent); } - private void throwExceptionForUnexpectedWeight(float weight, QueuePath queuePath, + private void throwExceptionForUnexpectedWeight(float weight, QueuePath queue, String label) { if ((weight < -1e-6 && Math.abs(weight + 1) > 1e-6) || weight > 10000) { throw new IllegalArgumentException( - "Illegal " + "weight=" + weight + " for queue=" + queuePath.getFullPath() + "label=" + "Illegal " + "weight=" + weight + " for queue=" + queue.getFullPath() + "label=" + label + ". Acceptable values: [0, 10000], -1 is same as not set"); } } - public float getNonLabeledQueueWeight(QueuePath queuePath) { - String configuredValue = get(getQueuePrefix(queuePath) + CAPACITY); + public float getNonLabeledQueueWeight(QueuePath queue) { + String configuredValue = get(getQueuePrefix(queue) + CAPACITY); float weight = extractFloatValueFromWeightConfig(configuredValue); - throwExceptionForUnexpectedWeight(weight, queuePath, ""); + throwExceptionForUnexpectedWeight(weight, queue, ""); return weight; } - public void setNonLabeledQueueWeight(QueuePath queuePath, float weight) { - set(getQueuePrefix(queuePath) + CAPACITY, weight + WEIGHT_SUFFIX); + public void setNonLabeledQueueWeight(QueuePath queue, float weight) { + set(getQueuePrefix(queue) + CAPACITY, weight + WEIGHT_SUFFIX); } - public void setLabeledQueueWeight(QueuePath queuePath, String label, float weight) { - set(getNodeLabelPrefix(queuePath, label) + CAPACITY, weight + WEIGHT_SUFFIX); + public void setLabeledQueueWeight(QueuePath queue, String label, float weight) { + set(getNodeLabelPrefix(queue, label) + CAPACITY, weight + WEIGHT_SUFFIX); } - public float getLabeledQueueWeight(QueuePath queuePath, String label) { - String configuredValue = get(getNodeLabelPrefix(queuePath, label) + CAPACITY); + public float getLabeledQueueWeight(QueuePath queue, String label) { + String configuredValue = get(getNodeLabelPrefix(queue, label) + CAPACITY); float weight = extractFloatValueFromWeightConfig(configuredValue); - throwExceptionForUnexpectedWeight(weight, queuePath, label); + throwExceptionForUnexpectedWeight(weight, queue, label); return weight; } - public float getNonLabeledQueueCapacity(QueuePath queuePath) { - String configuredCapacity = get(getQueuePrefix(queuePath) + CAPACITY); + public float getNonLabeledQueueCapacity(QueuePath queue) { + String configuredCapacity = get(getQueuePrefix(queue) + CAPACITY); boolean absoluteResourceConfigured = (configuredCapacity != null) && RESOURCE_PATTERN.matcher(configuredCapacity).find(); if (absoluteResourceConfigured || configuredWeightAsCapacity( @@ -562,10 +563,10 @@ public float getNonLabeledQueueCapacity(QueuePath queuePath) { // root.From AbstractCSQueue, absolute resource will be parsed and // updated. Once nodes are added/removed in cluster, capacity in // percentage will also be re-calculated. - return queuePath.isRoot() ? 100.0f : 0f; + return queue.isRoot() ? 100.0f : 0f; } - float capacity = queuePath.isRoot() + float capacity = queue.isRoot() ? 100.0f : (configuredCapacity == null) ? 0f @@ -573,39 +574,39 @@ public float getNonLabeledQueueCapacity(QueuePath queuePath) { if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException( - "Illegal " + "capacity of " + capacity + " for queue " + queuePath); + "Illegal " + "capacity of " + capacity + " for queue " + queue.getFullPath()); } LOG.debug("CSConf - getCapacity: queuePrefix={}, capacity={}", - getQueuePrefix(queuePath), capacity); + getQueuePrefix(queue), capacity); return capacity; } - public void setCapacity(QueuePath queuePath, float capacity) { - if (queuePath.getFullPath().equals("root")) { + public void setCapacity(QueuePath queue, float capacity) { + if (queue.getFullPath().equals("root")) { throw new IllegalArgumentException( "Cannot set capacity, root queue has a fixed capacity of 100.0f"); } - setFloat(getQueuePrefix(queuePath) + CAPACITY, capacity); + setFloat(getQueuePrefix(queue) + CAPACITY, capacity); LOG.debug("CSConf - setCapacity: queuePrefix={}, capacity={}", - getQueuePrefix(queuePath), capacity); + getQueuePrefix(queue), capacity); } @VisibleForTesting - public void setCapacity(QueuePath queuePath, String absoluteResourceCapacity) { - if (queuePath.getFullPath().equals("root")) { + public void setCapacity(QueuePath queue, String absoluteResourceCapacity) { + if (queue.getFullPath().equals("root")) { throw new IllegalArgumentException( "Cannot set capacity, root queue has a fixed capacity"); } - set(getQueuePrefix(queuePath) + CAPACITY, absoluteResourceCapacity); + set(getQueuePrefix(queue) + CAPACITY, absoluteResourceCapacity); LOG.debug("CSConf - setCapacity: queuePrefix={}, capacity={}", - getQueuePrefix(queuePath), absoluteResourceCapacity); + getQueuePrefix(queue), absoluteResourceCapacity); } - public float getNonLabeledQueueMaximumCapacity(QueuePath queuePath) { - String configuredCapacity = get(getQueuePrefix(queuePath) + MAXIMUM_CAPACITY); + public float getNonLabeledQueueMaximumCapacity(QueuePath queue) { + String configuredCapacity = get(getQueuePrefix(queue) + MAXIMUM_CAPACITY); boolean matcher = (configuredCapacity != null) && RESOURCE_PATTERN.matcher(configuredCapacity).find(); if (matcher) { @@ -625,38 +626,38 @@ public float getNonLabeledQueueMaximumCapacity(QueuePath queuePath) { return maxCapacity; } - public void setMaximumCapacity(QueuePath queuePath, float maxCapacity) { + public void setMaximumCapacity(QueuePath queue, float maxCapacity) { if (maxCapacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException("Illegal " + - "maximum-capacity of " + maxCapacity + " for queue " + queuePath.getFullPath()); + "maximum-capacity of " + maxCapacity + " for queue " + queue.getFullPath()); } - setFloat(getQueuePrefix(queuePath) + MAXIMUM_CAPACITY, maxCapacity); + setFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); LOG.debug("CSConf - setMaxCapacity: queuePrefix={}, maxCapacity={}", - getQueuePrefix(queuePath), maxCapacity); + getQueuePrefix(queue), maxCapacity); } - public void setCapacityByLabel(String queue, String label, float capacity) { + public void setCapacityByLabel(QueuePath queue, String label, float capacity) { setFloat(getNodeLabelPrefix(queue, label) + CAPACITY, capacity); } @VisibleForTesting - public void setCapacityByLabel(String queue, String label, + public void setCapacityByLabel(QueuePath queue, String label, String absoluteResourceCapacity) { set(getNodeLabelPrefix(queue, label) + CAPACITY, absoluteResourceCapacity); } - public void setMaximumCapacityByLabel(String queue, String label, + public void setMaximumCapacityByLabel(QueuePath queue, String label, float capacity) { setFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, capacity); } - public void setMaximumCapacityByLabel(String queue, String label, + public void setMaximumCapacityByLabel(QueuePath queue, String label, String absoluteResourceCapacity) { set(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, absoluteResourceCapacity); } - public float getUserLimit(String queue) { + public float getUserLimit(QueuePath queue) { float defaultUserLimit = getFloat(PREFIX + USER_LIMIT, DEFAULT_USER_LIMIT); float userLimit = getFloat(getQueuePrefix(queue) + USER_LIMIT, defaultUserLimit); @@ -668,7 +669,7 @@ public float getUserLimit(String queue) { // if possible. @SuppressWarnings("unchecked") public OrderingPolicy getAppOrderingPolicy( - String queue) { + QueuePath queue) { String policyType = get(getQueuePrefix(queue) + ORDERING_POLICY, DEFAULT_APP_ORDERING_POLICY); @@ -707,7 +708,7 @@ public OrderingPolicy getAppOrderingPolicy( return orderingPolicy; } - public void setUserLimit(String queue, float userLimit) { + public void setUserLimit(QueuePath queue, float userLimit) { setFloat(getQueuePrefix(queue) + USER_LIMIT, userLimit); LOG.debug("here setUserLimit: queuePrefix={}, userLimit={}", getQueuePrefix(queue), getUserLimit(queue)); @@ -718,7 +719,7 @@ public void setDefaultUserLimit(float defaultUserLimit) { setFloat(PREFIX + USER_LIMIT, defaultUserLimit); } - public float getUserLimitFactor(String queue) { + public float getUserLimitFactor(QueuePath queue) { float defaultUserLimitFactor = getFloat(PREFIX + USER_LIMIT_FACTOR, DEFAULT_USER_LIMIT_FACTOR); float userLimitFactor = getFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR, @@ -726,8 +727,8 @@ public float getUserLimitFactor(String queue) { return userLimitFactor; } - public void setUserLimitFactor(String queue, float userLimitFactor) { - setFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR, userLimitFactor); + public void setUserLimitFactor(QueuePath queuePath, float userLimitFactor) { + setFloat(getQueuePrefix(queuePath) + USER_LIMIT_FACTOR, userLimitFactor); } @VisibleForTesting @@ -735,7 +736,7 @@ public void setDefaultUserLimitFactor(float defaultUserLimitFactor) { setFloat(PREFIX + USER_LIMIT_FACTOR, defaultUserLimitFactor); } - public QueueState getConfiguredState(String queue) { + public QueueState getConfiguredState(QueuePath queue) { String state = get(getQueuePrefix(queue) + STATE); if (state == null) { return null; @@ -744,18 +745,18 @@ public QueueState getConfiguredState(String queue) { } } - public QueueState getState(String queue) { + public QueueState getState(QueuePath queue) { QueueState state = getConfiguredState(queue); return (state == null) ? QueueState.RUNNING : state; } @Private @VisibleForTesting - public void setState(String queue, QueueState state) { + public void setState(QueuePath queue, QueueState state) { set(getQueuePrefix(queue) + STATE, state.name()); } - public void setAccessibleNodeLabels(String queue, Set labels) { + public void setAccessibleNodeLabels(QueuePath queue, Set labels) { if (labels == null) { return; } @@ -763,27 +764,27 @@ public void setAccessibleNodeLabels(String queue, Set labels) { set(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS, str); } - public Set getAccessibleNodeLabels(String queue) { + public Set getAccessibleNodeLabels(QueuePath queue) { String accessibleLabelStr = get(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS); // When accessible-label is null, if (accessibleLabelStr == null) { // Only return null when queue is not ROOT - if (!queue.equals(ROOT)) { + if (!queue.isRoot()) { return null; } } else { // print a warning when accessibleNodeLabel specified in config and queue // is ROOT - if (queue.equals(ROOT)) { + if (queue.isRoot()) { LOG.warn("Accessible node labels for root queue will be ignored," + " it will be automatically set to \"*\"."); } } // always return ANY for queue root - if (queue.equals(ROOT)) { + if (queue.isRoot()) { return ImmutableSet.of(RMNodeLabelsManager.ANY); } @@ -821,7 +822,7 @@ private float extractFloatValueFromWeightConfig(String configureValue) { private float internalGetLabeledQueueCapacity(QueuePath queue, String label, String suffix, float defaultValue) { - String capacityPropertyName = getNodeLabelPrefix(queue.getFullPath(), label) + suffix; + String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix; String configuredCapacity = get(capacityPropertyName); boolean absoluteResourceConfigured = (configuredCapacity != null) && RESOURCE_PATTERN.matcher( @@ -846,7 +847,7 @@ private float internalGetLabeledQueueCapacity(QueuePath queue, String label, } if (LOG.isDebugEnabled()) { LOG.debug( - "CSConf - getCapacityOfLabel: prefix=" + getNodeLabelPrefix(queue.getFullPath(), + "CSConf - getCapacityOfLabel: prefix=" + getNodeLabelPrefix(queue, label) + ", capacity=" + capacity); } return capacity; @@ -860,7 +861,7 @@ public float getLabeledQueueMaximumCapacity(QueuePath queue, String label) { return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f); } - public String getDefaultNodeLabelExpression(String queue) { + public String getDefaultNodeLabelExpression(QueuePath queue) { String defaultLabelExpression = get(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION); if (defaultLabelExpression == null) { @@ -869,7 +870,7 @@ public String getDefaultNodeLabelExpression(String queue) { return defaultLabelExpression.trim(); } - public void setDefaultNodeLabelExpression(String queue, String exp) { + public void setDefaultNodeLabelExpression(QueuePath queue, String exp) { set(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION, exp); } @@ -877,12 +878,12 @@ public float getMaximumAMResourcePercentPerPartition(QueuePath queue, String label) { // If per-partition max-am-resource-percent is not configured, // use default value as max-am-resource-percent for this queue. - return getFloat(getNodeLabelPrefix(queue.getFullPath(), label) + return getFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_AM_RESOURCE_SUFFIX, - getMaximumApplicationMasterResourcePerQueuePercent(queue.getFullPath())); + getMaximumApplicationMasterResourcePerQueuePercent(queue)); } - public void setMaximumAMResourcePercentPerPartition(String queue, + public void setMaximumAMResourcePercentPerPartition(QueuePath queue, String label, float percent) { setFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_AM_RESOURCE_SUFFIX, percent); @@ -903,16 +904,16 @@ private static String getAclKey(QueueACL acl) { return "acl_" + StringUtils.toLowerCase(acl.toString()); } - public AccessControlList getAcl(String queue, QueueACL acl) { + public AccessControlList getAcl(QueuePath queue, QueueACL acl) { String queuePrefix = getQueuePrefix(queue); // The root queue defaults to all access if not defined // Sub queues inherit access if not defined - String defaultAcl = queue.equals(ROOT) ? ALL_ACL : NONE_ACL; + String defaultAcl = queue.isRoot() ? ALL_ACL : NONE_ACL; String aclString = get(queuePrefix + getAclKey(acl), defaultAcl); return new AccessControlList(aclString); } - public void setAcl(String queue, QueueACL acl, String aclString) { + public void setAcl(QueuePath queue, QueueACL acl, String aclString) { String queuePrefix = getQueuePrefix(queue); set(queuePrefix + getAclKey(acl), aclString); } @@ -932,9 +933,9 @@ private static String getAclKey(AccessType acl) { * @return A mapping of the queue ACLs. */ public Map getACLsForLegacyAutoCreatedLeafQueue( - String parentQueuePath) { + QueuePath parentQueuePath) { final String prefix = - getQueuePrefix(getAutoCreatedQueueTemplateConfPrefix( + getQueuePrefix(getAutoCreatedQueueObjectTemplateConfPrefix( parentQueuePath)); Map properties = new HashMap<>(); @@ -1014,12 +1015,12 @@ public Map getReservationAcls(String queue) { Map resAcls = new HashMap<>(); for (ReservationACL acl : ReservationACL.values()) { - resAcls.put(acl, getReservationAcl(queue, acl)); + resAcls.put(acl, getReservationAcl(new QueuePath(queue), acl)); } return resAcls; } - private AccessControlList getReservationAcl(String queue, ReservationACL + private AccessControlList getReservationAcl(QueuePath queue, ReservationACL acl) { String queuePrefix = getQueuePrefix(queue); // The root queue defaults to all access if not defined @@ -1029,17 +1030,17 @@ private AccessControlList getReservationAcl(String queue, ReservationACL return new AccessControlList(aclString); } - private void setAcl(String queue, ReservationACL acl, String aclString) { + private void setAcl(QueuePath queue, ReservationACL acl, String aclString) { String queuePrefix = getQueuePrefix(queue); set(queuePrefix + getAclKey(acl), aclString); } - private void setAcl(String queue, AccessType acl, String aclString) { + private void setAcl(QueuePath queue, AccessType acl, String aclString) { String queuePrefix = getQueuePrefix(queue); set(queuePrefix + getAclKey(acl), aclString); } - public Map getAcls(String queue) { + public Map getAcls(QueuePath queue) { Map acls = new HashMap(); for (QueueACL acl : QueueACL.values()) { @@ -1048,14 +1049,14 @@ public Map getAcls(String queue) { return acls; } - public void setAcls(String queue, Map acls) { + public void setAcls(QueuePath queue, Map acls) { for (Map.Entry e : acls.entrySet()) { setAcl(queue, e.getKey(), e.getValue().getAclString()); } } @VisibleForTesting - public void setReservationAcls(String queue, + public void setReservationAcls(QueuePath queue, Map acls) { for (Map.Entry e : acls.entrySet()) { setAcl(queue, e.getKey(), e.getValue().getAclString()); @@ -1063,7 +1064,7 @@ public void setReservationAcls(String queue, } @VisibleForTesting - public void setPriorityAcls(String queue, Priority priority, + public void setPriorityAcls(QueuePath queue, Priority priority, Priority defaultPriority, String[] acls) { StringBuilder aclString = new StringBuilder(); @@ -1080,7 +1081,7 @@ public void setPriorityAcls(String queue, Priority priority, setAcl(queue, AccessType.APPLICATION_MAX_PRIORITY, aclString.toString()); } - public List getPriorityAcls(String queue, + public List getPriorityAcls(QueuePath queue, Priority clusterMaxPriority) { String queuePrefix = getQueuePrefix(queue); String defaultAcl = ALL_ACL; @@ -1091,7 +1092,7 @@ public List getPriorityAcls(String queue, return priorityACLConfig.getPriorityAcl(clusterMaxPriority, aclString); } - public String[] getQueues(String queue) { + public String[] getQueues(QueuePath queue) { LOG.debug("CSConf - getQueues called for: queuePrefix={}", getQueuePrefix(queue)); String[] queues = getStrings(getQueuePrefix(queue) + QUEUES); @@ -1110,7 +1111,7 @@ public String[] getQueues(String queue) { return queues; } - public void setQueues(String queue, String[] subQueues) { + public void setQueues(QueuePath queue, String[] subQueues) { set(getQueuePrefix(queue) + QUEUES, StringUtils.arrayToString(subQueues)); LOG.debug("CSConf - setQueues: qPrefix={}, queues={}", getQueuePrefix(queue), StringUtils.arrayToString(subQueues)); @@ -1127,7 +1128,7 @@ public Resource getMinimumAllocation() { } @Private - public Priority getQueuePriority(String queue) { + public Priority getQueuePriority(QueuePath queue) { String queuePolicyPrefix = getQueuePrefix(queue); Priority pri = Priority.newInstance( getInt(queuePolicyPrefix + "priority", 0)); @@ -1135,7 +1136,7 @@ public Priority getQueuePriority(String queue) { } @Private - public void setQueuePriority(String queue, int priority) { + public void setQueuePriority(QueuePath queue, int priority) { String queuePolicyPrefix = getQueuePrefix(queue); setInt(queuePolicyPrefix + "priority", priority); } @@ -1148,7 +1149,7 @@ public void setQueuePriority(String queue, int priority) { * name of the queue * @return Resource object or Resource.none if not set */ - public Resource getQueueMaximumAllocation(String queue) { + public Resource getQueueMaximumAllocation(QueuePath queue) { String queuePrefix = getQueuePrefix(queue); String rawQueueMaxAllocation = get(queuePrefix + MAXIMUM_ALLOCATION, null); if (Strings.isNullOrEmpty(rawQueueMaxAllocation)) { @@ -1159,7 +1160,7 @@ public Resource getQueueMaximumAllocation(String queue) { } } - public void setQueueMaximumAllocation(String queue, String maximumAllocation) { + public void setQueueMaximumAllocation(QueuePath queue, String maximumAllocation) { String queuePrefix = getQueuePrefix(queue); set(queuePrefix + MAXIMUM_ALLOCATION, maximumAllocation); } @@ -1187,12 +1188,12 @@ public void reinitializeConfigurationProperties() { configurationProperties = new ConfigurationProperties(props); } - public long getQueueMaximumAllocationMb(String queue) { + public long getQueueMaximumAllocationMb(QueuePath queue) { String queuePrefix = getQueuePrefix(queue); return getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, (int)UNDEFINED); } - public int getQueueMaximumAllocationVcores(String queue) { + public int getQueueMaximumAllocationVcores(QueuePath queue) { String queuePrefix = getQueuePrefix(queue); return getInt(queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED); } @@ -1529,20 +1530,20 @@ void setWorkflowPriorityMappings( public boolean isReservable(String queue) { boolean isReservable = - getBoolean(getQueuePrefix(queue) + IS_RESERVABLE, false); + getBoolean(getQueuePrefix(new QueuePath(queue)) + IS_RESERVABLE, false); return isReservable; } public void setReservable(String queue, boolean isReservable) { - setBoolean(getQueuePrefix(queue) + IS_RESERVABLE, isReservable); + setBoolean(getQueuePrefix(new QueuePath(queue)) + IS_RESERVABLE, isReservable); LOG.debug("here setReservableQueue: queuePrefix={}, isReservableQueue={}", - getQueuePrefix(queue), isReservable(queue)); + getQueuePrefix(new QueuePath(queue)), isReservable(queue)); } @Override public long getReservationWindow(String queue) { long reservationWindow = - getLong(getQueuePrefix(queue) + RESERVATION_WINDOW, + getLong(getQueuePrefix(new QueuePath(queue)) + RESERVATION_WINDOW, DEFAULT_RESERVATION_WINDOW); return reservationWindow; } @@ -1550,7 +1551,7 @@ public long getReservationWindow(String queue) { @Override public float getAverageCapacity(String queue) { float avgCapacity = - getFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY, + getFloat(getQueuePrefix(new QueuePath(queue)) + AVERAGE_CAPACITY, MAXIMUM_CAPACITY_VALUE); return avgCapacity; } @@ -1558,33 +1559,33 @@ public float getAverageCapacity(String queue) { @Override public float getInstantaneousMaxCapacity(String queue) { float instMaxCapacity = - getFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY, + getFloat(getQueuePrefix(new QueuePath(queue)) + INSTANTANEOUS_MAX_CAPACITY, MAXIMUM_CAPACITY_VALUE); return instMaxCapacity; } - public void setInstantaneousMaxCapacity(String queue, float instMaxCapacity) { + public void setInstantaneousMaxCapacity(QueuePath queue, float instMaxCapacity) { setFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY, instMaxCapacity); } - public void setReservationWindow(String queue, long reservationWindow) { + public void setReservationWindow(QueuePath queue, long reservationWindow) { setLong(getQueuePrefix(queue) + RESERVATION_WINDOW, reservationWindow); } - public void setAverageCapacity(String queue, float avgCapacity) { + public void setAverageCapacity(QueuePath queue, float avgCapacity) { setFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY, avgCapacity); } @Override public String getReservationAdmissionPolicy(String queue) { String reservationPolicy = - get(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY, + get(getQueuePrefix(new QueuePath(queue)) + RESERVATION_ADMISSION_POLICY, DEFAULT_RESERVATION_ADMISSION_POLICY); return reservationPolicy; } - public void setReservationAdmissionPolicy(String queue, + public void setReservationAdmissionPolicy(QueuePath queue, String reservationPolicy) { set(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY, reservationPolicy); } @@ -1592,19 +1593,19 @@ public void setReservationAdmissionPolicy(String queue, @Override public String getReservationAgent(String queue) { String reservationAgent = - get(getQueuePrefix(queue) + RESERVATION_AGENT_NAME, + get(getQueuePrefix(new QueuePath(queue)) + RESERVATION_AGENT_NAME, DEFAULT_RESERVATION_AGENT_NAME); return reservationAgent; } - public void setReservationAgent(String queue, String reservationPolicy) { + public void setReservationAgent(QueuePath queue, String reservationPolicy) { set(getQueuePrefix(queue) + RESERVATION_AGENT_NAME, reservationPolicy); } @Override public boolean getShowReservationAsQueues(String queuePath) { boolean showReservationAsQueues = - getBoolean(getQueuePrefix(queuePath) + getBoolean(getQueuePrefix(new QueuePath(queuePath)) + RESERVATION_SHOW_RESERVATION_AS_QUEUE, DEFAULT_SHOW_RESERVATIONS_AS_QUEUES); return showReservationAsQueues; @@ -1613,7 +1614,7 @@ public boolean getShowReservationAsQueues(String queuePath) { @Override public String getReplanner(String queue) { String replanner = - get(getQueuePrefix(queue) + RESERVATION_PLANNER_NAME, + get(getQueuePrefix(new QueuePath(queue)) + RESERVATION_PLANNER_NAME, DEFAULT_RESERVATION_PLANNER_NAME); return replanner; } @@ -1621,7 +1622,7 @@ public String getReplanner(String queue) { @Override public boolean getMoveOnExpiry(String queue) { boolean killOnExpiry = - getBoolean(getQueuePrefix(queue) + RESERVATION_MOVE_ON_EXPIRY, + getBoolean(getQueuePrefix(new QueuePath(queue)) + RESERVATION_MOVE_ON_EXPIRY, DEFAULT_RESERVATION_MOVE_ON_EXPIRY); return killOnExpiry; } @@ -1629,7 +1630,7 @@ public boolean getMoveOnExpiry(String queue) { @Override public long getEnforcementWindow(String queue) { long enforcementWindow = - getLong(getQueuePrefix(queue) + RESERVATION_ENFORCEMENT_WINDOW, + getLong(getQueuePrefix(new QueuePath(queue)) + RESERVATION_ENFORCEMENT_WINDOW, DEFAULT_RESERVATION_ENFORCEMENT_WINDOW); return enforcementWindow; } @@ -1642,7 +1643,7 @@ public long getEnforcementWindow(String queue) { * @param queue queue path * @param preemptionDisabled true if preemption is disabled on queue */ - public void setPreemptionDisabled(String queue, boolean preemptionDisabled) { + public void setPreemptionDisabled(QueuePath queue, boolean preemptionDisabled) { setBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED, preemptionDisabled); } @@ -1655,7 +1656,7 @@ public void setPreemptionDisabled(String queue, boolean preemptionDisabled) { * is not set in the configuration * @return true if preemption is disabled on queue, false otherwise */ - public boolean getPreemptionDisabled(String queue, boolean defaultVal) { + public boolean getPreemptionDisabled(QueuePath queue, boolean defaultVal) { boolean preemptionDisabled = getBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED, defaultVal); @@ -1670,7 +1671,7 @@ public boolean getPreemptionDisabled(String queue, boolean defaultVal) { * configuration * @return true if preemption is disabled on queue, false otherwise */ - public boolean getIntraQueuePreemptionDisabled(String queue, + public boolean getIntraQueuePreemptionDisabled(QueuePath queue, boolean defaultVal) { return getBoolean(getQueuePrefix(queue) + INTRA_QUEUE_PREEMPTION_CONFIG_PREFIX @@ -1680,7 +1681,7 @@ public boolean getIntraQueuePreemptionDisabled(String queue, /** * Get configured node labels in a given queuePath */ - public Set getConfiguredNodeLabels(String queuePath) { + public Set getConfiguredNodeLabels(QueuePath queuePath) { Set configuredNodeLabels = new HashSet(); Entry e = null; @@ -1751,7 +1752,7 @@ public Priority getClusterLevelApplicationMaxPriority() { YarnConfiguration.DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY)); } - public Integer getDefaultApplicationPriorityConfPerQueue(String queue) { + public Integer getDefaultApplicationPriorityConfPerQueue(QueuePath queue) { Integer defaultPriority = getInt(getQueuePrefix(queue) + DEFAULT_APPLICATION_PRIORITY, DEFAULT_CONFIGURATION_APPLICATION_PRIORITY); @@ -1759,12 +1760,12 @@ public Integer getDefaultApplicationPriorityConfPerQueue(String queue) { } @VisibleForTesting - public void setOrderingPolicy(String queue, String policy) { + public void setOrderingPolicy(QueuePath queue, String policy) { set(getQueuePrefix(queue) + ORDERING_POLICY, policy); } @VisibleForTesting - public void setOrderingPolicyParameter(String queue, + public void setOrderingPolicyParameter(QueuePath queue, String parameterKey, String parameterValue) { set(getQueuePrefix(queue) + ORDERING_POLICY + "." + parameterKey, parameterValue); @@ -1778,7 +1779,7 @@ public static boolean shouldAppFailFast(Configuration conf) { return conf.getBoolean(APP_FAIL_FAST, DEFAULT_APP_FAIL_FAST); } - public Integer getMaxParallelAppsForQueue(String queue) { + public Integer getMaxParallelAppsForQueue(QueuePath queue) { int defaultMaxParallelAppsForQueue = getInt(PREFIX + MAX_PARALLEL_APPLICATIONS, DEFAULT_MAX_PARALLEL_APPLICATIONS); @@ -1803,12 +1804,12 @@ public Integer getMaxParallelAppsForUser(String user) { : defaultMaxParallelAppsForUser; } - public boolean getAllowZeroCapacitySum(String queue) { + public boolean getAllowZeroCapacitySum(QueuePath queue) { return getBoolean(getQueuePrefix(queue) + ALLOW_ZERO_CAPACITY_SUM, DEFAULT_ALLOW_ZERO_CAPACITY_SUM); } - public void setAllowZeroCapacitySum(String queue, boolean value) { + public void setAllowZeroCapacitySum(QueuePath queue, boolean value) { setBoolean(getQueuePrefix(queue) + ALLOW_ZERO_CAPACITY_SUM, value); } @@ -2006,12 +2007,12 @@ public void setGlobalMaximumApplicationsPerQueue(int val) { @Private - public void setQueueOrderingPolicy(String queue, String policy) { + public void setQueueOrderingPolicy(QueuePath queue, String policy) { set(getQueuePrefix(queue) + ORDERING_POLICY, policy); } @Private - public QueueOrderingPolicy getQueueOrderingPolicy(String queue, + public QueueOrderingPolicy getQueueOrderingPolicy(QueuePath queue, String parentPolicy) { String defaultPolicy = parentPolicy; if (null == defaultPolicy) { @@ -2033,7 +2034,7 @@ public QueueOrderingPolicy getQueueOrderingPolicy(String queue, qop = (QueueOrderingPolicy) Class.forName(policyType).newInstance(); } catch (Exception e) { String message = "Unable to construct queue ordering policy=" - + policyType + " queue=" + queue; + + policyType + " queue=" + queue.getFullPath(); throw new YarnRuntimeException(message, e); } } @@ -2152,7 +2153,7 @@ public void setPUOrderingPolicyUnderUtilizedPreemptionMoveReservation( * @param queuePath full queue path * @return map of user weights, if they exist. Otherwise, return empty map. */ - public UserWeights getAllUserWeightsForQueue(String queuePath) { + public UserWeights getAllUserWeightsForQueue(QueuePath queuePath) { return UserWeights.createByConfig(this, getConfigurationProperties(), queuePath); } @@ -2170,23 +2171,23 @@ public int getMaxAssignPerHeartbeat() { public static final String DEFAULT_LIFETIME_SUFFIX = "default-application-lifetime"; - public long getMaximumLifetimePerQueue(String queue) { + public long getMaximumLifetimePerQueue(QueuePath queue) { long maximumLifetimePerQueue = getLong( getQueuePrefix(queue) + MAXIMUM_LIFETIME_SUFFIX, (long) UNDEFINED); return maximumLifetimePerQueue; } - public void setMaximumLifetimePerQueue(String queue, long maximumLifetime) { + public void setMaximumLifetimePerQueue(QueuePath queue, long maximumLifetime) { setLong(getQueuePrefix(queue) + MAXIMUM_LIFETIME_SUFFIX, maximumLifetime); } - public long getDefaultLifetimePerQueue(String queue) { + public long getDefaultLifetimePerQueue(QueuePath queue) { long maximumLifetimePerQueue = getLong( getQueuePrefix(queue) + DEFAULT_LIFETIME_SUFFIX, (long) UNDEFINED); return maximumLifetimePerQueue; } - public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) { + public void setDefaultLifetimePerQueue(QueuePath queue, long defaultLifetime) { setLong(getQueuePrefix(queue) + DEFAULT_LIFETIME_SUFFIX, defaultLifetime); } @@ -2247,7 +2248,7 @@ public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) { * is false */ @Private - public boolean isAutoCreateChildQueueEnabled(String queuePath) { + public boolean isAutoCreateChildQueueEnabled(QueuePath queuePath) { boolean isAutoCreateEnabled = getBoolean( getQueuePrefix(queuePath) + AUTO_CREATE_CHILD_QUEUE_ENABLED, DEFAULT_AUTO_CREATE_CHILD_QUEUE_ENABLED); @@ -2256,21 +2257,21 @@ public boolean isAutoCreateChildQueueEnabled(String queuePath) { @Private @VisibleForTesting - public void setAutoCreateChildQueueEnabled(String queuePath, + public void setAutoCreateChildQueueEnabled(QueuePath queuePath, boolean autoCreationEnabled) { setBoolean(getQueuePrefix(queuePath) + AUTO_CREATE_CHILD_QUEUE_ENABLED, autoCreationEnabled); } - public void setAutoQueueCreationV2Enabled(String queuePath, + public void setAutoQueueCreationV2Enabled(QueuePath queuePath, boolean autoQueueCreation) { setBoolean( getQueuePrefix(queuePath) + AUTO_QUEUE_CREATION_V2_ENABLED, autoQueueCreation); } - public boolean isAutoQueueCreationV2Enabled(String queuePath) { + public boolean isAutoQueueCreationV2Enabled(QueuePath queuePath) { boolean isAutoQueueCreation = getBoolean( getQueuePrefix(queuePath) + AUTO_QUEUE_CREATION_V2_ENABLED, DEFAULT_AUTO_QUEUE_CREATION_ENABLED); @@ -2294,7 +2295,7 @@ public boolean isAutoQueueCreationV2Enabled(String queuePath) { */ @Private public boolean getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded( - String queuePath) { + QueuePath queuePath) { boolean shouldFailAutoQueueCreationOnExceedingGuaranteedCapacity = getBoolean(getQueuePrefix(queuePath) + FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY, @@ -2305,7 +2306,7 @@ public boolean getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded( @VisibleForTesting @Private public void setShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded( - String queuePath, boolean autoCreationEnabled) { + QueuePath queuePath, boolean autoCreationEnabled) { setBoolean( getQueuePrefix(queuePath) + FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY, @@ -2320,7 +2321,7 @@ public void setShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded( * @return the max number of leaf queues allowed to be auto created */ @Private - public int getAutoCreatedQueuesMaxChildQueuesLimit(String queuePath) { + public int getAutoCreatedQueuesMaxChildQueuesLimit(QueuePath queuePath) { return getInt(getQueuePrefix(queuePath) + AUTO_CREATE_QUEUE_MAX_QUEUES, DEFAULT_AUTO_CREATE_QUEUE_MAX_QUEUES); @@ -2335,14 +2336,14 @@ public int getAutoCreatedQueuesMaxChildQueuesLimit(String queuePath) { * in new auto created. */ @Private - public int getAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath) { + public int getAutoCreatedQueuesV2MaxChildQueuesLimit(QueuePath queuePath) { return getInt(getQueuePrefix(queuePath) + AUTO_QUEUE_CREATION_V2_MAX_QUEUES, DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES); } @VisibleForTesting - public void setAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath, + public void setAutoCreatedQueuesV2MaxChildQueuesLimit(QueuePath queuePath, int maxQueues) { setInt(getQueuePrefix(queuePath) + AUTO_QUEUE_CREATION_V2_MAX_QUEUES, maxQueues); @@ -2399,7 +2400,7 @@ public void setAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath, * is true. */ @Private - public boolean isAutoExpiredDeletionEnabled(String queuePath) { + public boolean isAutoExpiredDeletionEnabled(QueuePath queuePath) { boolean isAutoExpiredDeletionEnabled = getBoolean( getQueuePrefix(queuePath) + AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE, @@ -2409,7 +2410,7 @@ public boolean isAutoExpiredDeletionEnabled(String queuePath) { @Private @VisibleForTesting - public void setAutoExpiredDeletionEnabled(String queuePath, + public void setAutoExpiredDeletionEnabled(QueuePath queuePath, boolean autoRemovalEnable) { setBoolean(getQueuePrefix(queuePath) + AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE, @@ -2447,7 +2448,7 @@ public long getAutoExpiredDeletionTime() { * @return Configured policy class name */ @Private - public String getAutoCreatedQueueManagementPolicy(String queue) { + public String getAutoCreatedQueueManagementPolicy(QueuePath queue) { String autoCreatedQueueManagementPolicy = get(getQueuePrefix(queue) + AUTO_CREATED_QUEUE_MANAGEMENT_POLICY, DEFAULT_AUTO_CREATED_QUEUE_MANAGEMENT_POLICY); @@ -2458,19 +2459,19 @@ public String getAutoCreatedQueueManagementPolicy(String queue) { * Get The policy class configured to manage capacities for auto created leaf * queues under the specified parent * - * @param queueName The parent queue's name + * @param queue The parent queue's name * @return The policy class configured to manage capacities for auto created * leaf queues under the specified parent queue */ @Private protected AutoCreatedQueueManagementPolicy getAutoCreatedQueueManagementPolicyClass( - String queueName) { + QueuePath queue) { String queueManagementPolicyClassName = - getAutoCreatedQueueManagementPolicy(queueName); + getAutoCreatedQueueManagementPolicy(queue); LOG.info("Using Auto Created Queue Management Policy: " - + queueManagementPolicyClassName + " for queue: " + queueName); + + queueManagementPolicyClassName + " for queue: " + queue.getFullPath()); try { Class queueManagementPolicyClazz = getClassByName( queueManagementPolicyClassName); @@ -2486,36 +2487,36 @@ public String getAutoCreatedQueueManagementPolicy(String queue) { } catch (ClassNotFoundException e) { throw new YarnRuntimeException( "Could not instantiate " + "AutoCreatedQueueManagementPolicy: " - + queueManagementPolicyClassName + " for queue: " + queueName, + + queueManagementPolicyClassName + " for queue: " + queue.getFullPath(), e); } } @VisibleForTesting @Private - public void setAutoCreatedLeafQueueConfigCapacity(String queuePath, + public void setAutoCreatedLeafQueueConfigCapacity(QueuePath queuePath, float val) { - String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( + QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix( queuePath); setCapacity(leafQueueConfPrefix, val); } @VisibleForTesting @Private - public void setAutoCreatedLeafQueueTemplateCapacityByLabel(String queuePath, + public void setAutoCreatedLeafQueueTemplateCapacityByLabel(QueuePath queuePath, String label, float val) { - String leafQueueConfPrefix = - getAutoCreatedQueueTemplateConfPrefix(queuePath); + QueuePath leafQueueConfPrefix = + getAutoCreatedQueueObjectTemplateConfPrefix(queuePath); setCapacityByLabel(leafQueueConfPrefix, label, val); } @VisibleForTesting @Private - public void setAutoCreatedLeafQueueTemplateCapacityByLabel(String queuePath, + public void setAutoCreatedLeafQueueTemplateCapacityByLabel(QueuePath queuePath, String label, Resource resource) { - String leafQueueConfPrefix = - getAutoCreatedQueueTemplateConfPrefix(queuePath); + QueuePath leafQueueConfPrefix = + getAutoCreatedQueueObjectTemplateConfPrefix(queuePath); StringBuilder resourceString = new StringBuilder(); @@ -2541,18 +2542,18 @@ public void setAutoCreatedLeafQueueConfigMaxCapacity(QueuePath queuePath, @Private @VisibleForTesting - public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath, + public void setAutoCreatedLeafQueueTemplateMaxCapacity(QueuePath queuePath, String label, float val) { - String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( + QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix( queuePath); setMaximumCapacityByLabel(leafQueueConfPrefix, label, val); } @Private @VisibleForTesting - public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath, + public void setAutoCreatedLeafQueueTemplateMaxCapacity(QueuePath queuePath, String label, Resource resource) { - String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( + QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix( queuePath); StringBuilder resourceString = new StringBuilder(); @@ -2570,37 +2571,37 @@ public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath, @VisibleForTesting @Private - public void setAutoCreatedLeafQueueConfigUserLimit(String queuePath, + public void setAutoCreatedLeafQueueConfigUserLimit(QueuePath queuePath, int val) { - String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( + QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix( queuePath); setUserLimit(leafQueueConfPrefix, val); } @VisibleForTesting @Private - public void setAutoCreatedLeafQueueConfigUserLimitFactor(String queuePath, + public void setAutoCreatedLeafQueueConfigUserLimitFactor(QueuePath queuePath, float val) { - String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( + QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix( queuePath); setUserLimitFactor(leafQueueConfPrefix, val); } @Private @VisibleForTesting - public void setAutoCreatedLeafQueueConfigDefaultNodeLabelExpression(String + public void setAutoCreatedLeafQueueConfigDefaultNodeLabelExpression(QueuePath queuePath, String expression) { - String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( + QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix( queuePath); setDefaultNodeLabelExpression(leafQueueConfPrefix, expression); } @Private @VisibleForTesting - public void setAutoCreatedLeafQueueConfigMaximumAllocation(String + public void setAutoCreatedLeafQueueConfigMaximumAllocation(QueuePath queuePath, String expression) { - String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix( + QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix( queuePath); setQueueMaximumAllocation(leafQueueConfPrefix, expression); } @@ -2629,7 +2630,7 @@ public static String getUnits(String resourceValue) { * Resource types * @return ResourceInformation */ - public Resource getMinimumResourceRequirement(String label, String queue, + public Resource getMinimumResourceRequirement(String label, QueuePath queue, Set resourceTypes) { return internalGetLabeledResourceRequirementForQueue(queue, label, resourceTypes, CAPACITY); @@ -2646,7 +2647,7 @@ public Resource getMinimumResourceRequirement(String label, String queue, * Resource types * @return Resource */ - public Resource getMaximumResourceRequirement(String label, String queue, + public Resource getMaximumResourceRequirement(String label, QueuePath queue, Set resourceTypes) { return internalGetLabeledResourceRequirementForQueue(queue, label, resourceTypes, MAXIMUM_CAPACITY); @@ -2665,10 +2666,11 @@ public void setMaximumResourceRequirement(String label, QueuePath queue, } public Map parseConfiguredResourceVector( - String queuePath, Set labels) { + QueuePath queuePath, Set labels) { Map queueResourceVectors = new HashMap<>(); for (String label : labels) { - queueResourceVectors.put(label, queueCapacityConfigParser.parse(this, queuePath, label)); + queueResourceVectors.put(label, queueCapacityConfigParser.parse(this, + queuePath, label)); } return queueResourceVectors; @@ -2691,15 +2693,15 @@ private void updateMinMaxResourceToConf(String label, QueuePath queue, + ResourceUtils. getCustomResourcesStrings(resource) + "]"); - String prefix = getQueuePrefix(queue.getFullPath()) + type; + String prefix = getQueuePrefix(queue) + type; if (!label.isEmpty()) { - prefix = getQueuePrefix(queue.getFullPath()) + ACCESSIBLE_NODE_LABELS + DOT + label + prefix = getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT + type; } set(prefix, resourceString.toString()); } - public boolean checkConfigTypeIsAbsoluteResource(String label, String queue, + public boolean checkConfigTypeIsAbsoluteResource(String label, QueuePath queue, Set resourceTypes) { String propertyName = getNodeLabelPrefix(queue, label) + CAPACITY; String resourceString = get(propertyName); @@ -2714,7 +2716,7 @@ public boolean checkConfigTypeIsAbsoluteResource(String label, String queue, return false; } - private Resource internalGetLabeledResourceRequirementForQueue(String queue, + private Resource internalGetLabeledResourceRequirementForQueue(QueuePath queue, String label, Set resourceTypes, String suffix) { String propertyName = getNodeLabelPrefix(queue, label) + suffix; String resourceString = get(propertyName); @@ -2828,7 +2830,7 @@ private void updateResourceValuesFromConfig(Set resourceTypes, public static final boolean DEFAULT_MULTI_NODE_PLACEMENT_ENABLED = false; public String getMultiNodesSortingAlgorithmPolicy( - String queue) { + QueuePath queue) { String policyName = get( getQueuePrefix(queue) + "multi-node-sorting.policy"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index e74ac47dc841c..d344a064a7b27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -236,45 +236,48 @@ static CSQueue parseQueue( CSQueue parent, String queueName, CSQueueStore newQueues, CSQueueStore oldQueues, QueueHook hook) throws IOException { CSQueue queue; - String fullQueueName = (parent == null) ? queueName : - (QueuePath.createFromQueues(parent.getQueuePath(), queueName).getFullPath()); - String[] staticChildQueueNames = conf.getQueues(fullQueueName); + QueuePath fullQueuePath = (parent == null) ? new QueuePath(queueName) : + (QueuePath.createFromQueues(parent.getQueuePath(), queueName)); + String[] staticChildQueueNames = conf.getQueues(fullQueuePath); List childQueueNames = staticChildQueueNames != null ? Arrays.asList(staticChildQueueNames) : Collections.emptyList(); - CSQueue oldQueue = oldQueues.get(fullQueueName); + CSQueue oldQueue = oldQueues.get(fullQueuePath.getFullPath()); - boolean isReservableQueue = conf.isReservable(fullQueueName); - boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(fullQueueName); + boolean isReservableQueue = conf.isReservable(fullQueuePath.getFullPath()); + boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(fullQueuePath); // if a queue is eligible for auto queue creation v2 it must be a ParentQueue // (even if it is empty) final boolean isDynamicParent = oldQueue instanceof ParentQueue && oldQueue.isDynamicQueue(); boolean isAutoQueueCreationEnabledParent = isDynamicParent || conf.isAutoQueueCreationV2Enabled( - fullQueueName) || isAutoCreateEnabled; + fullQueuePath) || isAutoCreateEnabled; if (childQueueNames.size() == 0 && !isAutoQueueCreationEnabledParent) { validateParent(parent, queueName); // Check if the queue will be dynamically managed by the Reservation system if (isReservableQueue) { - queue = new PlanQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName)); + queue = new PlanQueue(queueContext, queueName, parent, + oldQueues.get(fullQueuePath.getFullPath())); ReservationQueue defaultResQueue = ((PlanQueue) queue).initializeDefaultInternalQueue(); newQueues.add(defaultResQueue); } else { - queue = new LeafQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName)); + queue = new LeafQueue(queueContext, queueName, parent, + oldQueues.get(fullQueuePath.getFullPath())); } queue = hook.hook(queue); } else { if (isReservableQueue) { - throw new IllegalStateException("Only Leaf Queues can be reservable for " + fullQueueName); + throw new IllegalStateException("Only Leaf Queues can be reservable for " + + fullQueuePath.getFullPath()); } ParentQueue parentQueue; if (isAutoCreateEnabled) { parentQueue = new ManagedParentQueue(queueContext, queueName, parent, oldQueues.get( - fullQueueName)); + fullQueuePath.getFullPath())); } else { parentQueue = new ParentQueue(queueContext, queueName, parent, oldQueues.get( - fullQueueName)); + fullQueuePath.getFullPath())); } queue = hook.hook(parentQueue); @@ -293,7 +296,7 @@ static CSQueue parseQueue( newQueues.add(queue); - LOG.info("Initialized queue: " + fullQueueName); + LOG.info("Initialized queue: " + fullQueuePath.getFullPath()); return queue; } @@ -319,7 +322,7 @@ private void updateQueues(CSQueueStore existingQueues, boolean isRemovable = isDanglingDynamicQueue || !isDynamicQueue(queue) && newQueues.get(queue.getQueuePath()) == null && !(queue instanceof AutoCreatedLeafQueue && - conf.isAutoCreateChildQueueEnabled(queue.getParent().getQueuePath())); + conf.isAutoCreateChildQueueEnabled(queue.getParent().getQueuePathObject())); if (isRemovable) { existingQueues.remove(queue); @@ -505,7 +508,7 @@ public AbstractLeafQueue createQueue(QueuePath queue) CSQueue parentQueue = getQueue(parentQueueName); if (parentQueue != null && csContext.getConfiguration() - .isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) { + .isAutoCreateChildQueueEnabled(parentQueue.getQueuePathObject())) { return createLegacyAutoQueue(queue); } else { return createAutoQueue(queue); @@ -621,7 +624,7 @@ public List getPermissionsForDynamicQueue( if (parentQueue instanceof AbstractManagedParentQueue) { // An AbstractManagedParentQueue must have been found for Legacy AQC permissions.add(new Permission(privilegedEntity, - csConf.getACLsForLegacyAutoCreatedLeafQueue(queuePath.getParent()))); + csConf.getACLsForLegacyAutoCreatedLeafQueue(queuePath.getParentObject()))); } else { // Every other case must be a Flexible Leaf Queue permissions.add(new Permission(privilegedEntity, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 32a33bff5a4e0..1859ab8d954d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -60,7 +60,7 @@ public ManagedParentQueue(final CapacitySchedulerQueueContext queueContext, shouldFailAutoCreationWhenGuaranteedCapacityExceeded = queueContext.getConfiguration() .getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded( - getQueuePath()); + getQueuePathObject()); leafQueueTemplate = initializeLeafQueueConfigs().build(); @@ -78,7 +78,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) shouldFailAutoCreationWhenGuaranteedCapacityExceeded = queueContext.getConfiguration() .getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded( - getQueuePath()); + getQueuePathObject()); //validate if capacity is exceeded for child queues if (shouldFailAutoCreationWhenGuaranteedCapacityExceeded) { @@ -128,7 +128,7 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) private void initializeQueueManagementPolicy() throws IOException { queueManagementPolicy = queueContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass( - getQueuePath()); + getQueuePathObject()); queueManagementPolicy.init(this); } @@ -136,7 +136,7 @@ private void initializeQueueManagementPolicy() throws IOException { private void reinitializeQueueManagementPolicy() throws IOException { AutoCreatedQueueManagementPolicy managementPolicy = queueContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass( - getQueuePath()); + getQueuePathObject()); if (!(managementPolicy.getClass().equals( this.queueManagementPolicy.getClass()))) { @@ -165,7 +165,7 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws setAbsoluteResourceTemplates(configuration, queueResourceQuotas); QueuePath templateQueuePath = QueuePrefixes - .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath()); + .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()); Set templateConfiguredNodeLabels = queueContext .getQueueManager().getConfiguredNodeLabelsForAllQueues() .getLabelsByQueue(templateQueuePath.getFullPath()); @@ -193,14 +193,14 @@ protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() throws private void setAbsoluteResourceTemplates(CapacitySchedulerConfiguration configuration, QueueResourceQuotas queueResourceQuotas) throws IOException { QueuePath templateQueuePath = QueuePrefixes - .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath()); + .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()); Set templateConfiguredNodeLabels = queueContext .getQueueManager().getConfiguredNodeLabelsForAllQueues() .getLabelsByQueue(templateQueuePath.getFullPath()); for (String nodeLabel : templateConfiguredNodeLabels) { Resource templateMinResource = configuration.getMinimumResourceRequirement( - nodeLabel, templateQueuePath.getFullPath(), resourceTypes); + nodeLabel, templateQueuePath, resourceTypes); queueResourceQuotas.setConfiguredMinResource(nodeLabel, templateMinResource); if (this.capacityConfigType.equals(CapacityConfigType.PERCENTAGE) @@ -222,14 +222,14 @@ private void updateQueueCapacities(QueueCapacities queueCapacities) { configuration.getMinimumResourceRequirement( label, QueuePrefixes - .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), + .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()), resourceTypes), getQueueResourceQuotas().getConfiguredMinResource(label))); Resource childMaxResource = configuration .getMaximumResourceRequirement(label, QueuePrefixes - .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()), + .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()), resourceTypes); Resource parentMaxRes = getQueueResourceQuotas() .getConfiguredMaxResource(label); @@ -289,7 +289,7 @@ public void addChildQueue(CSQueue childQueue) String leafQueuePath = childQueue.getQueuePath(); int maxQueues = conf.getAutoCreatedQueuesMaxChildQueuesLimit( - parentQueue.getQueuePath()); + parentQueue.getQueuePathObject()); if (parentQueue.getChildQueues().size() >= maxQueues) { throw new SchedulerDynamicEditException( @@ -380,7 +380,7 @@ public List getAllApplications() { public String getLeafQueueConfigPrefix() { return CapacitySchedulerConfiguration.PREFIX + QueuePrefixes - .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()); + .getAutoCreatedQueueTemplateConfPrefix(getQueuePathObject()); } public boolean shouldFailAutoCreationWhenGuaranteedCapacityExceeded() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 0949d512a79f1..6a45dcfc7274e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -135,7 +135,7 @@ private ParentQueue(CapacitySchedulerQueueContext queueContext, this.childQueues = new ArrayList<>(); this.allowZeroCapacitySum = queueContext.getConfiguration() - .getAllowZeroCapacitySum(getQueuePath()); + .getAllowZeroCapacitySum(getQueuePathObject()); setupQueueConfigs(queueContext.getClusterResource()); } @@ -170,7 +170,7 @@ protected void setupQueueConfigs(Resource clusterResource) // Initialize queue ordering policy queueOrderingPolicy = configuration.getQueueOrderingPolicy( - getQueuePath(), parent == null ? + getQueuePathObject(), parent == null ? null : ((ParentQueue) parent).getQueueOrderingPolicyConfigName()); queueOrderingPolicy.setQueues(childQueues); @@ -536,7 +536,7 @@ private CSQueue addDynamicChildQueue(String childQueuePath, boolean isLeaf) // Check if the max queue limit is exceeded. int maxQueues = queueContext.getConfiguration(). - getAutoCreatedQueuesV2MaxChildQueuesLimit(getQueuePath()); + getAutoCreatedQueuesV2MaxChildQueuesLimit(getQueuePathObject()); if (childQueues.size() >= maxQueues) { throw new SchedulerDynamicEditException( "Cannot auto create queue " + childQueuePath + ". Max Child " @@ -620,7 +620,7 @@ public void removeChildQueue(CSQueue queue) */ public boolean isEligibleForAutoQueueCreation() { return isDynamicQueue() || queueContext.getConfiguration(). - isAutoQueueCreationV2Enabled(getQueuePath()); + isAutoQueueCreationV2Enabled(getQueuePathObject()); } @Override @@ -1655,7 +1655,7 @@ Map getEffectiveMinRatio(String label) { public boolean isEligibleForAutoDeletion() { return isDynamicQueue() && getChildQueues().size() == 0 && queueContext.getConfiguration(). - isAutoExpiredDeletionEnabled(this.getQueuePath()); + isAutoExpiredDeletionEnabled(this.getQueuePathObject()); } public AutoCreatedQueueTemplate getAutoCreatedQueueTemplate() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index eab94436d7f8e..e8b270ef2a232 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -61,8 +61,8 @@ public PlanQueue(CapacitySchedulerQueueContext queueContext, String queueName, DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * super .getAbsoluteCapacity()); } - float configuredUserLimit = conf.getUserLimit(queuePath); - float configuredUserLimitFactor = conf.getUserLimitFactor(queuePath); + float configuredUserLimit = conf.getUserLimit(queuePathObject); + float configuredUserLimitFactor = conf.getUserLimitFactor(queuePathObject); int configuredMaxAppsPerUserForReservation = (int) (maxAppsForReservation * (configuredUserLimit / 100.0f) * configuredUserLimitFactor); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java index 101c8076fdc04..7028423e8c227 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java @@ -36,7 +36,7 @@ public QueueAllocationSettings(Resource minimumAllocation) { this.minimumAllocation = minimumAllocation; } - void setupMaximumAllocation(CapacitySchedulerConfiguration configuration, String queuePath, + void setupMaximumAllocation(CapacitySchedulerConfiguration configuration, QueuePath queuePath, CSQueue parent) { Resource clusterMax = ResourceUtils .fetchMaximumAllocationFromConfig(configuration); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java index 2e71d8eef3edb..9218c877e71b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java @@ -37,7 +37,7 @@ public class QueueAppLifetimeAndLimitSettings { public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration, AbstractCSQueue q, QueuePath queuePath) { // Store max parallel apps property - this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath.getFullPath()); + this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath); this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration); this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration, maxApplicationLifetime); @@ -45,7 +45,7 @@ public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configura private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguration conf) { CSQueue parentQ = q.getParent(); - long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath()); + long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePathObject()); // If q is the root queue, then get max app lifetime from conf. if (q.getQueuePathObject().isRoot()) { @@ -64,7 +64,7 @@ private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguratio private long setupInheritedDefaultAppLifetime(CSQueue q, QueuePath queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) { CSQueue parentQ = q.getParent(); - long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath.getFullPath()); + long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath); defaultAppLifetimeWasSpecifiedInConfig = (defaultAppLifetime >= 0 || (!queuePath.isRoot() && diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java index c431d2bb45543..adf5f6e0ffcb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java @@ -53,7 +53,7 @@ private void initializeNodeLabels(CapacitySchedulerConfiguration configuration, } private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) { - this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath()); + this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath); // Inherit labels from parent if not set if (this.accessibleLabels == null && parent != null) { this.accessibleLabels = parent.getAccessibleNodeLabels(); @@ -62,7 +62,7 @@ private void initializeAccessibleLabels(CapacitySchedulerConfiguration configura private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) { this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression( - queuePath.getFullPath()); + queuePath); // If the accessible labels is not null and the queue has a parent with a // similar set of labels copy the defaultNodeLabelExpression from the parent if (this.accessibleLabels != null && parent != null @@ -83,7 +83,7 @@ private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration confi } } else { // Fallback to suboptimal but correct logic - this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath.getFullPath()); + this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index 440742b908927..808f9765fc01f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -60,6 +60,16 @@ public QueuePath(String fullPath) { setFromFullPath(fullPath); } + /** + * Static method to create Queue path from parent QueuePath object and a leaf queue name. + * @param parent Parent QueuePath object for the queue + * @param leaf Name of the leaf queue + * @return QueuePath object + */ + public static QueuePath addQueueToPath(QueuePath parent, String leaf) { + return new QueuePath(parent.getFullPath(), leaf); + } + /** * Constructor to create Queue path from queue names. * The provided queue names will be concatenated by dots, giving a full queue path. @@ -113,6 +123,14 @@ public String getParent() { return parent; } + /** + * Getter for the parent object of the path. + * @return Parent QueuePath object of the queue, null if there is no parent. + */ + public QueuePath getParentObject() { + return hasParent() ? new QueuePath(parent) : null; + } + /** * Getter for the leaf part of the path. * @return The leaf queue name diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java index 5ec7d01bce9b1..7e561c6fe0741 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java @@ -42,7 +42,7 @@ private QueueStateHelper() {} public static void setQueueState(AbstractCSQueue queue) { QueueState previousState = queue.getState(); QueueState configuredState = queue.getQueueContext().getConfiguration().getConfiguredState( - queue.getQueuePath()); + queue.getQueuePathObject()); QueueState parentState = (queue.getParent() == null) ? null : queue.getParent().getState(); // verify that we can not any value for State other than RUNNING/STOPPED diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java index 41c02204c852e..6535de1c61d8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java @@ -43,7 +43,7 @@ public static UserWeights createEmpty() { public static UserWeights createByConfig( CapacitySchedulerConfiguration conf, ConfigurationProperties configurationProperties, - String queuePath) { + QueuePath queuePath) { String queuePathPlusPrefix = getQueuePrefix(queuePath) + USER_SETTINGS; Map props = configurationProperties .getPropertiesWithPrefix(queuePathPlusPrefix); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java index 88c93019680cb..99c1c22d0a081 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java @@ -20,7 +20,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; @@ -76,19 +78,20 @@ private static void removeQueue( throw new IOException("Queue " + queueToRemove + " not found"); } siblingQueues.remove(queueName); - String parentQueuePath = queueToRemove.substring(0, queueToRemove + String parentPath = queueToRemove.substring(0, queueToRemove .lastIndexOf('.')); + QueuePath parentQueuePath = new QueuePath(parentPath); proposedConf.setQueues(parentQueuePath, siblingQueues.toArray( new String[0])); String queuesConfig = CapacitySchedulerConfiguration.PREFIX - + parentQueuePath + CapacitySchedulerConfiguration.DOT + + parentPath + CapacitySchedulerConfiguration.DOT + CapacitySchedulerConfiguration.QUEUES; if (siblingQueues.isEmpty()) { confUpdate.put(queuesConfig, null); // Unset Ordering Policy of Leaf Queue converted from // Parent Queue after removeQueue String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX - + parentQueuePath + CapacitySchedulerConfiguration.DOT + + parentPath + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY; proposedConf.unset(queueOrderingPolicy); confUpdate.put(queueOrderingPolicy, null); @@ -117,15 +120,16 @@ private static void addQueue( queueName)) { throw new IOException("Can't add existing queue " + queuePath); } - String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.')); - String[] siblings = proposedConf.getQueues(parentQueue); + String parentPath = queuePath.substring(0, queuePath.lastIndexOf('.')); + QueuePath parentQueuePath = new QueuePath(parentPath); + String[] siblings = proposedConf.getQueues(parentQueuePath); List siblingQueues = siblings == null ? new ArrayList<>() : new ArrayList<>(Arrays.asList(siblings)); siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1)); - proposedConf.setQueues(parentQueue, + proposedConf.setQueues(parentQueuePath, siblingQueues.toArray(new String[0])); confUpdate.put(CapacitySchedulerConfiguration.PREFIX - + parentQueue + CapacitySchedulerConfiguration.DOT + + parentPath + CapacitySchedulerConfiguration.DOT + CapacitySchedulerConfiguration.QUEUES, Joiner.on(',').join(siblingQueues)); String keyPrefix = CapacitySchedulerConfiguration.PREFIX @@ -143,7 +147,7 @@ private static void addQueue( // Unset Ordering Policy of Parent Queue converted from // Leaf Queue after addQueue String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX - + parentQueue + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY; + + parentPath + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY; if (siblingQueues.size() == 1) { proposedConf.unset(queueOrderingPolicy); confUpdate.put(queueOrderingPolicy, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java index c2af5c5a576f6..1eb8cac84ebd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java @@ -21,6 +21,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.util.UnitsConversionUtil; @@ -68,9 +69,9 @@ public QueueCapacityConfigParser() { * @return a parsed capacity vector */ public QueueCapacityVector parse(CapacitySchedulerConfiguration conf, - String queuePath, String label) { + QueuePath queuePath, String label) { - if (queuePath.equals(CapacitySchedulerConfiguration.ROOT)) { + if (queuePath.isRoot()) { return QueueCapacityVector.of(100f, QueueCapacityType.PERCENTAGE); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java index 6ff19e21a6e71..3d6797e648185 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.MappingRulesDescription; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.Rule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.Rule.FallbackResult; @@ -121,7 +122,7 @@ private void handleNestedRule(List rules, PlacementRule parentRule = userRule.getParentRule(); boolean parentCreate = ((FSPlacementRule) parentRule).getCreateFlag(); Policy policy; - String queueName = null; + QueuePath queueName = null; if (parentRule instanceof PrimaryGroupPlacementRule) { policy = Policy.PRIMARY_GROUP_USER; @@ -130,7 +131,7 @@ private void handleNestedRule(List rules, } else if (parentRule instanceof DefaultPlacementRule) { DefaultPlacementRule defaultRule = (DefaultPlacementRule) parentRule; policy = Policy.USER; - queueName = defaultRule.defaultQueueName; + queueName = new QueuePath(defaultRule.defaultQueueName); } else { throw new IllegalArgumentException( "Unsupported parent nested rule: " @@ -187,7 +188,7 @@ private Rule createNestedRule(Policy policy, boolean create, FSConfigToCSConfigRuleHandler ruleHandler, boolean fsParentCreate, - String parentQueue, + QueuePath parentQueue, CapacitySchedulerConfiguration csConf, boolean usePercentages) { @@ -196,7 +197,7 @@ private Rule createNestedRule(Policy policy, // "parent" is already set to "root" at this point, // so we override it if necessary if (parentQueue != null) { - rule.setParentQueue(parentQueue); + rule.setParentQueue(parentQueue.getFullPath()); } if (usePercentages) { @@ -207,13 +208,13 @@ private Rule createNestedRule(Policy policy, } else if (policy == Policy.SECONDARY_GROUP_USER) { ruleHandler.handleFSParentCreateFlag("root."); } else { - ruleHandler.handleFSParentCreateFlag(parentQueue); + ruleHandler.handleFSParentCreateFlag(parentQueue.getFullPath()); } } // check if parent conflicts with existing static queues if (create && policy == Policy.USER) { - ruleHandler.handleRuleAutoCreateFlag(parentQueue); + ruleHandler.handleRuleAutoCreateFlag(parentQueue.getFullPath()); checkStaticDynamicConflict(parentQueue, csConf, ruleHandler); } } else { @@ -230,7 +231,7 @@ private Rule createNestedRule(Policy policy, return rule; } - private void checkStaticDynamicConflict(String parentPath, + private void checkStaticDynamicConflict(QueuePath parentPath, CapacitySchedulerConfiguration csConf, FSConfigToCSConfigRuleHandler ruleHandler) { String[] childQueues = csConf.getQueues(parentPath); @@ -238,7 +239,7 @@ private void checkStaticDynamicConflict(String parentPath, // User must be warned: static + dynamic queues are under the // same parent if (childQueues != null && childQueues.length > 0) { - ruleHandler.handleChildStaticDynamicConflict(parentPath); + ruleHandler.handleChildStaticDynamicConflict(parentPath.getFullPath()); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 83394b15a37d1..ccb77140489b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper; import java.util.ArrayList; @@ -49,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected float normalizedWeight; protected String queueName; private String queuePath; + private QueuePath queuePathObject; protected int maxParallelApps; private boolean isAbsoluteResource; protected CapacitySchedulerQueueInfoList queues; @@ -76,6 +78,7 @@ public CapacitySchedulerInfo() { public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { this.queueName = parent.getQueueName(); this.queuePath = parent.getQueuePath(); + this.queuePathObject = parent.getQueuePathObject(); this.usedCapacity = parent.getUsedCapacity() * 100; this.capacity = parent.getCapacity() * 100; float max = parent.getMaximumCapacity(); @@ -97,7 +100,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { CapacitySchedulerConfiguration conf = cs.getConfiguration(); queueAcls = new QueueAclsInfo(); - queueAcls.addAll(getSortedQueueAclInfoList(parent, queueName, conf)); + queueAcls.addAll(getSortedQueueAclInfoList(parent, queuePathObject, conf)); queuePriority = parent.getPriority().getPriority(); if (parent instanceof ParentQueue) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index c07297c9cff36..4df37b26143df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper; @@ -54,6 +55,7 @@ public class CapacitySchedulerQueueInfo { static final float EPSILON = 1e-8f; protected String queuePath; + protected QueuePath queuePathObject; protected float capacity; protected float usedCapacity; protected float maxCapacity; @@ -102,6 +104,7 @@ public class CapacitySchedulerQueueInfo { CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) { queuePath = q.getQueuePath(); + queuePathObject = q.getQueuePathObject(); capacity = q.getCapacity() * 100; usedCapacity = q.getUsedCapacity() * 100; @@ -158,7 +161,7 @@ public class CapacitySchedulerQueueInfo { CapacitySchedulerConfiguration conf = cs.getConfiguration(); queueAcls = new QueueAclsInfo(); - queueAcls.addAll(getSortedQueueAclInfoList(q, queuePath, conf)); + queueAcls.addAll(getSortedQueueAclInfoList(q, queuePathObject, conf)); queuePriority = q.getPriority().getPriority(); if (q instanceof ParentQueue) { @@ -180,12 +183,12 @@ public class CapacitySchedulerQueueInfo { AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE; autoCreateChildQueueEnabled = conf. - isAutoCreateChildQueueEnabled(queuePath); - leafQueueTemplate = new LeafQueueTemplateInfo(conf, queuePath); + isAutoCreateChildQueueEnabled(queuePathObject); + leafQueueTemplate = new LeafQueueTemplateInfo(conf, queuePathObject); } public static ArrayList getSortedQueueAclInfoList( - CSQueue queue, String queuePath, CapacitySchedulerConfiguration conf) { + CSQueue queue, QueuePath queuePath, CapacitySchedulerConfiguration conf) { ArrayList queueAclsInfo = new ArrayList<>(); for (Map.Entry e : ((AbstractCSQueue) queue).getACLs().entrySet()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java index 577dd519a0fa9..98112012b7ec9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java @@ -19,6 +19,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import javax.xml.bind.annotation.XmlAccessType; @@ -41,7 +42,7 @@ public class LeafQueueTemplateInfo { public LeafQueueTemplateInfo() { } // JAXB needs this - public LeafQueueTemplateInfo(Configuration conf, String queuePath) { + public LeafQueueTemplateInfo(Configuration conf, QueuePath queuePath) { String configPrefix = QueuePrefixes. getQueuePrefix(queuePath) + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + DOT; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java index a011c8a8b35ab..00591fe1b532b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java @@ -22,6 +22,8 @@ import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -47,6 +49,13 @@ public abstract class ACLsTestBase { protected static final String QUEUEA = "queueA"; protected static final String QUEUEB = "queueB"; protected static final String QUEUEC = "queueC"; + protected static QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + protected static QueuePath A_QUEUE_PATH = new QueuePath( + CapacitySchedulerConfiguration.ROOT + "." + QUEUEA); + protected static QueuePath B_QUEUE_PATH = new QueuePath( + CapacitySchedulerConfiguration.ROOT + "." + QUEUEB); + protected static QueuePath C_QUEUE_PATH = new QueuePath( + CapacitySchedulerConfiguration.ROOT + "." + QUEUEC); protected static final Logger LOG = LoggerFactory.getLogger(TestApplicationACLs.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java index 45060e25be421..496bdbad8437d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; @@ -499,19 +500,15 @@ private boolean checkCapacity(Collection plans) { private static Configuration createCapacitySchedulerConfiguration() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { + csConf.setQueues(ROOT, new String[] { QUEUEA, QUEUEB, QUEUEC }); - String absoluteQueueA = CapacitySchedulerConfiguration.ROOT + "." + QUEUEA; - String absoluteQueueB = CapacitySchedulerConfiguration.ROOT + "." + QUEUEB; - String absoluteQueueC = CapacitySchedulerConfiguration.ROOT + "." + QUEUEC; - - csConf.setCapacity(absoluteQueueA, 50f); - csConf.setCapacity(absoluteQueueB, 20f); - csConf.setCapacity(absoluteQueueC, 30f); - csConf.setReservable(absoluteQueueA, true); - csConf.setReservable(absoluteQueueB, true); - csConf.setReservable(absoluteQueueC, true); + csConf.setCapacity(A_QUEUE_PATH, 50f); + csConf.setCapacity(B_QUEUE_PATH, 20f); + csConf.setCapacity(C_QUEUE_PATH, 30f); + csConf.setReservable(A_QUEUE_PATH.getFullPath(), true); + csConf.setReservable(B_QUEUE_PATH.getFullPath(), true); + csConf.setReservable(C_QUEUE_PATH.getFullPath(), true); // Set up ACLs on Queue A Map reservationAclsOnQueueA = @@ -528,7 +525,7 @@ private static Configuration createCapacitySchedulerConfiguration() { reservationAclsOnQueueA.put(ReservationACL.LIST_RESERVATIONS, listACLonQueueA); - csConf.setReservationAcls(absoluteQueueA, reservationAclsOnQueueA); + csConf.setReservationAcls(A_QUEUE_PATH, reservationAclsOnQueueA); // Set up ACLs on Queue B Map reservationAclsOnQueueB = @@ -545,7 +542,7 @@ private static Configuration createCapacitySchedulerConfiguration() { reservationAclsOnQueueB.put(ReservationACL.LIST_RESERVATIONS, listACLonQueueB); - csConf.setReservationAcls(absoluteQueueB, reservationAclsOnQueueB); + csConf.setReservationAcls(B_QUEUE_PATH, reservationAclsOnQueueB); csConf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); csConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index bdfd58ef1407b..8470cd48b8d1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -116,7 +116,6 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getAutoCreatedQueueTemplateConfPrefix; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix; import static org.junit.Assert.assertTrue; @@ -151,9 +150,12 @@ public class TestAppManager extends AppManagerTestBase{ private ResourceScheduler scheduler; private static final String USER_ID_PREFIX = "userid="; - private static final String ROOT_PARENT = PREFIX + "root.parent."; - private static final QueuePath TEST_QUEUE_PATH = new QueuePath("root.test"); - private static final QueuePath MANAGED_PARENT_QUEUE_PATH = new QueuePath("root.managedparent"); + private static final String ROOT_PARENT_PATH = PREFIX + "root.parent."; + private static final QueuePath ROOT = new QueuePath("root"); + private static final QueuePath DEFAULT = new QueuePath("root.default"); + private static final QueuePath TEST = new QueuePath("root.test"); + private static final QueuePath PARENT = new QueuePath("root.parent"); + private static final QueuePath MANAGED_PARENT = new QueuePath("root.managedparent"); public synchronized RMAppEventType getAppEventType() { return appEventType; @@ -325,11 +327,11 @@ public void testQueueSubmitWithACLsEnabledWithQueueMapping() CapacitySchedulerConfiguration(conf, false); csConf.set(PREFIX + "root.queues", "default,test"); - csConf.setCapacity(DEFAULT_QUEUE_PATH, 50.0f); - csConf.setMaximumCapacity(DEFAULT_QUEUE_PATH, 100.0f); + csConf.setCapacity(DEFAULT, 50.0f); + csConf.setMaximumCapacity(DEFAULT, 100.0f); - csConf.setCapacity(TEST_QUEUE_PATH, 50.0f); - csConf.setMaximumCapacity(TEST_QUEUE_PATH, 100.0f); + csConf.setCapacity(TEST, 50.0f); + csConf.setMaximumCapacity(TEST, 100.0f); csConf.set(PREFIX + "root.acl_submit_applications", " "); csConf.set(PREFIX + "root.acl_administer_queue", " "); @@ -367,11 +369,11 @@ public void testQueueSubmitWithACLsEnabledWithQueueMappingForLegacyAutoCreatedQu conf, false); csConf.set(PREFIX + "root.queues", "default,managedparent"); - csConf.setCapacity(DEFAULT_QUEUE_PATH, 50.0f); - csConf.setMaximumCapacity(DEFAULT_QUEUE_PATH, 100.0f); + csConf.setCapacity(DEFAULT, 50.0f); + csConf.setMaximumCapacity(DEFAULT, 100.0f); - csConf.setCapacity(MANAGED_PARENT_QUEUE_PATH, 50.0f); - csConf.setMaximumCapacity(MANAGED_PARENT_QUEUE_PATH, 100.0f); + csConf.setCapacity(MANAGED_PARENT, 50.0f); + csConf.setMaximumCapacity(MANAGED_PARENT, 100.0f); csConf.set(PREFIX + "root.acl_submit_applications", " "); csConf.set(PREFIX + "root.acl_administer_queue", " "); @@ -382,9 +384,9 @@ public void testQueueSubmitWithACLsEnabledWithQueueMappingForLegacyAutoCreatedQu csConf.set(PREFIX + "root.managedparent.acl_administer_queue", "admin"); csConf.set(PREFIX + "root.managedparent.acl_submit_applications", "user1"); - csConf.setAutoCreateChildQueueEnabled("root.managedparent", true); - csConf.setAutoCreatedLeafQueueConfigCapacity("root.managedparent", 30f); - csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.managedparent", 100f); + csConf.setAutoCreateChildQueueEnabled(MANAGED_PARENT, true); + csConf.setAutoCreatedLeafQueueConfigCapacity(MANAGED_PARENT, 30f); + csConf.setAutoCreatedLeafQueueConfigMaxCapacity(MANAGED_PARENT, 100f); MockRM newMockRM = new MockRM(csConf); CapacityScheduler cs = @@ -427,13 +429,13 @@ public void testLegacyAutoCreatedQueuesWithACLTemplates() csConf.set(PREFIX + "root.acl_submit_applications", " "); csConf.set(PREFIX + "root.acl_administer_queue", " "); - csConf.setCapacity(parentQueuePath.getFullPath(), 100.0f); + csConf.setCapacity(parentQueuePath, 100.0f); csConf.set(PREFIX + "root.parent.acl_administer_queue", "user1,user4"); csConf.set(PREFIX + "root.parent.acl_submit_applications", "user1,user4"); - csConf.setAutoCreateChildQueueEnabled(parentQueuePath.getFullPath(), true); - csConf.setAutoCreatedLeafQueueConfigCapacity(parentQueuePath.getFullPath(), 50f); - csConf.setAutoCreatedLeafQueueConfigMaxCapacity(parentQueuePath.getFullPath(), 100f); + csConf.setAutoCreateChildQueueEnabled(parentQueuePath, true); + csConf.setAutoCreatedLeafQueueConfigCapacity(parentQueuePath, 50f); + csConf.setAutoCreatedLeafQueueConfigMaxCapacity(parentQueuePath, 100f); String autoCreatedQueuePrefix = getAutoCreatedQueueTemplateConfPrefix(parentQueuePath); QueuePath autoCreatedQueuePath = new QueuePath(autoCreatedQueuePrefix); @@ -513,18 +515,18 @@ public void testFlexibleAutoCreatedQueuesWithSpecializedACLTemplatesAndDynamicPa YarnConfiguration conf = createYarnACLEnabledConfiguration(); CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf); - csConf.set(ROOT_PARENT + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity", "1w"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_administer_queue", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_administer_queue", "user2"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_submit_applications", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_submit_applications", "user2"); - csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity", + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity", "1w"); - csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue", + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue", "user3"); - csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications", + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications", "user3"); MockRM newMockRM = new MockRM(csConf); @@ -604,18 +606,18 @@ public void testFlexibleAutoCreatedQueuesWithMixedCommonLeafACLTemplatesAndDynam YarnConfiguration conf = createYarnACLEnabledConfiguration(); CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf); - csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity", "1w"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue", "user2"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications", "user2"); - csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity", + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity", "1w"); - csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue", + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue", "user3"); - csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications", + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications", "user3"); testFlexibleAQCDWithMixedTemplatesDynamicParentACLScenario(conf, csConf); @@ -627,18 +629,18 @@ public void testFlexibleAutoCreatedQueuesWithMixedCommonCommonACLTemplatesAndDyn YarnConfiguration conf = createYarnACLEnabledConfiguration(); CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf); - csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity", "1w"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue", "user2"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications", "user2"); - csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity", + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity", "1w"); - csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue", + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue", "user3"); - csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications", + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications", "user3"); testFlexibleAQCDWithMixedTemplatesDynamicParentACLScenario(conf, csConf); @@ -727,11 +729,11 @@ public void testFlexibleAutoCreatedQueuesWithACLTemplatesALeafOnly() YarnConfiguration conf = createYarnACLEnabledConfiguration(); CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf); - csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity", "1w"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue", "user2"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications", "user2"); testFlexibleAQCLeafOnly(conf, csConf); @@ -743,11 +745,11 @@ public void testFlexibleAutoCreatedQueuesWithSpecialisedACLTemplatesALeafOnly() YarnConfiguration conf = createYarnACLEnabledConfiguration(); CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf); - csConf.set(ROOT_PARENT + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity", "1w"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue", "user2"); - csConf.set(ROOT_PARENT + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications", + csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications", "user2"); testFlexibleAQCLeafOnly(conf, csConf); @@ -827,7 +829,7 @@ public void testFlexibleAutoCreatedQueuesWithSpecializedACLTemplatesAndDynamicRo csConf.set(PREFIX + "root.acl_submit_applications", "user1"); csConf.set(PREFIX + "root.acl_administer_queue", "admin1"); - csConf.setAutoQueueCreationV2Enabled("root", true); + csConf.setAutoQueueCreationV2Enabled(ROOT, true); csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity", "1w"); @@ -926,7 +928,7 @@ public void testFlexibleAutoCreatedQueuesMultiLevelDynamicParentACL() csConf.set(PREFIX + "root.acl_submit_applications", "user1"); csConf.set(PREFIX + "root.acl_administer_queue", "admin1"); - csConf.setAutoQueueCreationV2Enabled("root", true); + csConf.setAutoQueueCreationV2Enabled(ROOT, true); csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity", "1w"); @@ -1004,11 +1006,11 @@ private CapacitySchedulerConfiguration createFlexibleAQCBaseACLConfiguration( csConf.set(PREFIX + "root.acl_submit_applications", " "); csConf.set(PREFIX + "root.acl_administer_queue", " "); - csConf.setCapacity("root.parent", "1w"); + csConf.setCapacity(PARENT, "1w"); csConf.set(PREFIX + "root.parent.acl_administer_queue", "user1"); csConf.set(PREFIX + "root.parent.acl_submit_applications", "user1"); - csConf.setAutoQueueCreationV2Enabled("root.parent", true); + csConf.setAutoQueueCreationV2Enabled(PARENT, true); return csConf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java index 4c261ab8aaaf3..dd1f295758d9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -55,8 +56,6 @@ import java.util.List; import java.util.Set; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_QUEUE_PATH; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_QUEUE_PATH; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils.toSet; import static org.junit.Assert.fail; @@ -262,20 +261,23 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); - - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - conf.setCapacity(A_QUEUE_PATH, 50); - conf.setMaximumCapacity(A_QUEUE_PATH, 100); + final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + conf.setQueues(ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(ROOT, "x", 100); + conf.setCapacityByLabel(ROOT, "y", 100); + + final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + final QueuePath A = new QueuePath(A_PATH); + conf.setCapacity(A, 50); + conf.setMaximumCapacity(A, 100); conf.setAccessibleNodeLabels(A, toSet("x")); conf.setDefaultNodeLabelExpression(A, "x"); conf.setCapacityByLabel(A, "x", 100); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; - conf.setCapacity(B_QUEUE_PATH, 50); - conf.setMaximumCapacity(B_QUEUE_PATH, 100); + final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + final QueuePath B = new QueuePath(B_PATH); + conf.setCapacity(B, 50); + conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y")); conf.setDefaultNodeLabelExpression(B, "y"); conf.setCapacityByLabel(B, "y", 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 52a34fbf76161..2bbdbb5ec0230 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .TestCapacitySchedulerAutoCreatedQueueBase; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt; @@ -567,46 +568,48 @@ private void checkFSQueue(ResourceManager rm, private static final String QUEUE_DOESNT_EXIST = "NoSuchQueue"; private static final String USER_1 = "user1"; private static final String USER_2 = "user2"; + private static final String Q_R_PATH = CapacitySchedulerConfiguration.ROOT + "." + R; + private static final String Q_A_PATH = Q_R_PATH + "." + A; + private static final String Q_B_PATH = Q_R_PATH + "." + B; + private static final String Q_B1_PATH = Q_B_PATH + "." + B1; + private static final String Q_B2_PATH = Q_B_PATH + "." + B2; + + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath R_QUEUE_PATH = new QueuePath(Q_R_PATH); + private static final QueuePath A_QUEUE_PATH = new QueuePath(Q_A_PATH); + private static final QueuePath B_QUEUE_PATH = new QueuePath(Q_B_PATH); + private static final QueuePath B1_QUEUE_PATH = new QueuePath(Q_B1_PATH); + private static final QueuePath B2_QUEUE_PATH = new QueuePath(Q_B2_PATH); private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { R }); - final String Q_R = CapacitySchedulerConfiguration.ROOT + "." + R; - conf.setCapacity(Q_R, 100); - final String Q_A = Q_R + "." + A; - final String Q_B = Q_R + "." + B; - conf.setQueues(Q_R, new String[] {A, B}); - conf.setCapacity(Q_A, 50); - conf.setCapacity(Q_B, 50); + conf.setQueues(ROOT, new String[] { R }); + conf.setCapacity(R_QUEUE_PATH, 100); + conf.setQueues(R_QUEUE_PATH, new String[] {A, B}); + conf.setCapacity(A_QUEUE_PATH, 50); + conf.setCapacity(B_QUEUE_PATH, 50); conf.setDouble(CapacitySchedulerConfiguration .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.5f); } private void setupQueueConfigurationOnlyA( CapacitySchedulerConfiguration conf) { - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { R }); - final String Q_R = CapacitySchedulerConfiguration.ROOT + "." + R; - conf.setCapacity(Q_R, 100); - final String Q_A = Q_R + "." + A; - conf.setQueues(Q_R, new String[] {A}); - conf.setCapacity(Q_A, 100); + conf.setQueues(ROOT, new String[] { R }); + conf.setCapacity(R_QUEUE_PATH, 10); + conf.setQueues(R_QUEUE_PATH, new String[] {A}); + conf.setCapacity(A_QUEUE_PATH, 100); conf.setDouble(CapacitySchedulerConfiguration .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 1.0f); } private void setupQueueConfigurationChildOfB(CapacitySchedulerConfiguration conf) { - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { R }); - final String Q_R = CapacitySchedulerConfiguration.ROOT + "." + R; - conf.setCapacity(Q_R, 100); - final String Q_A = Q_R + "." + A; - final String Q_B = Q_R + "." + B; - final String Q_B1 = Q_B + "." + B1; - final String Q_B2 = Q_B + "." + B2; - conf.setQueues(Q_R, new String[] {A, B}); - conf.setCapacity(Q_A, 50); - conf.setCapacity(Q_B, 50); - conf.setQueues(Q_B, new String[] {B1, B2}); - conf.setCapacity(Q_B1, 50); - conf.setCapacity(Q_B2, 50); + conf.setQueues(ROOT, new String[] { R }); + conf.setCapacity(R_QUEUE_PATH, 100); + conf.setQueues(R_QUEUE_PATH, new String[] {A, B}); + conf.setCapacity(A_QUEUE_PATH, 50); + conf.setCapacity(B_QUEUE_PATH, 50); + conf.setQueues(B_QUEUE_PATH, new String[] {B1, B2}); + conf.setCapacity(B1_QUEUE_PATH, 50); + conf.setCapacity(B2_QUEUE_PATH, 50); conf.setDouble(CapacitySchedulerConfiguration .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.5f); } @@ -636,9 +639,9 @@ public void testRMRestartWithRemovedQueue() throws Exception{ MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{QUEUE_DOESNT_EXIST}); + csConf.setQueues(ROOT, new String[]{QUEUE_DOESNT_EXIST}); final String noQueue = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_DOESNT_EXIST; - csConf.setCapacity(noQueue, 100); + csConf.setCapacity(new QueuePath(noQueue), 100); rm2 = new MockRM(csConf, memStore); rm2.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index fe89a698cf2e6..5bf7685770d42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; @@ -127,6 +128,14 @@ public class TestProportionalCapacityPreemptionPolicy { ApplicationId.newInstance(TS, 4), 0); final ArgumentCaptor evtCaptor = ArgumentCaptor.forClass(ContainerPreemptEvent.class); + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath QUEUE_A = new QueuePath("root.queueA"); + private static final QueuePath QUEUE_A_QUEUE_B = new QueuePath("root.queueA.queueB"); + private static final QueuePath QUEUE_B = new QueuePath("root.queueB"); + private static final QueuePath QUEUE_D = new QueuePath("root.queueD"); + private static final QueuePath QUEUE_D_QUEUE_E = new QueuePath( "root.queueD.queueE"); + private static final QueuePath QUEUE_A_QUEUE_C = new QueuePath( "root.queueA.queueC"); + public enum priority { AMCONTAINER(0), CONTAINER(1), LABELEDCONTAINER(2); @@ -346,7 +355,7 @@ public void testPerQueueDisablePreemption() { { 3, 0, 0, 0 }, // subqueues }; - conf.setPreemptionDisabled("root.queueB", true); + conf.setPreemptionDisabled(QUEUE_B, true); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); @@ -359,7 +368,7 @@ public void testPerQueueDisablePreemption() { // event handler will count only events from the following test and not the // previous one. setup(); - conf.setPreemptionDisabled("root.queueB", false); + conf.setPreemptionDisabled(QUEUE_B, false); ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData); policy2.editSchedule(); @@ -398,7 +407,7 @@ public void testPerQueueDisablePreemptionHierarchical() { // Need to call setup() again to reset mDisp setup(); // Turn off preemption for queueB and it's children - conf.setPreemptionDisabled("root.queueA.queueB", true); + conf.setPreemptionDisabled(QUEUE_A_QUEUE_B, true); ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData); policy2.editSchedule(); ApplicationAttemptId expectedAttemptOnQueueC = @@ -445,7 +454,7 @@ public void testPerQueueDisablePreemptionBroadHierarchical() { // Need to call setup() again to reset mDisp setup(); // Turn off preemption for queueB(appA) - conf.setPreemptionDisabled("root.queueA.queueB", true); + conf.setPreemptionDisabled(QUEUE_A_QUEUE_B, true); ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData); policy2.editSchedule(); // Now that queueB(appA) is not preemptable, verify that resources come @@ -455,8 +464,8 @@ public void testPerQueueDisablePreemptionBroadHierarchical() { setup(); // Turn off preemption for two of the 3 queues with over-capacity. - conf.setPreemptionDisabled("root.queueD.queueE", true); - conf.setPreemptionDisabled("root.queueA.queueB", true); + conf.setPreemptionDisabled(QUEUE_D_QUEUE_E, true); + conf.setPreemptionDisabled(QUEUE_A_QUEUE_B, true); ProportionalCapacityPreemptionPolicy policy3 = buildPolicy(qData); policy3.editSchedule(); @@ -497,7 +506,7 @@ public void testPerQueueDisablePreemptionInheritParent() { // Turn off preemption for queueA and it's children. queueF(appC)'s request // should starve. setup(); // Call setup() to reset mDisp - conf.setPreemptionDisabled("root.queueA", true); + conf.setPreemptionDisabled(QUEUE_A, true); ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData); policy2.editSchedule(); verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA))); // queueC @@ -521,7 +530,7 @@ public void testPerQueuePreemptionNotAllUntouchable() { { -1, -1, 1, 1, 1, -1, 1, 1, 1 }, // req granularity { 2, 3, 0, 0, 0, 3, 0, 0, 0 }, // subqueues }; - conf.setPreemptionDisabled("root.queueA.queueC", true); + conf.setPreemptionDisabled(QUEUE_A_QUEUE_C, true); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // Although queueC(appB) is way over capacity and is untouchable, @@ -545,7 +554,7 @@ public void testPerQueueDisablePreemptionRootDisablesAll() { { 3, 2, 0, 0, 2, 0, 0, 2, 0, 0 }, // subqueues }; - conf.setPreemptionDisabled("root", true); + conf.setPreemptionDisabled(ROOT, true); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // All queues should be non-preemptable, so request should starve. @@ -572,7 +581,7 @@ public void testPerQueueDisablePreemptionOverAbsMaxCapacity() { { 2, 2, 0, 0, 2, 0, 0 }, // subqueues }; // QueueE inherits non-preemption from QueueD - conf.setPreemptionDisabled("root.queueD", true); + conf.setPreemptionDisabled(QUEUE_D, true); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // appC is running on QueueE. QueueE is over absMaxCap, but is not @@ -1325,7 +1334,7 @@ private boolean mockPreemptionStatus(String queuePathName) { String qName = ""; while(tokenizer.hasMoreTokens()) { qName += tokenizer.nextToken(); - preemptionDisabled = conf.getPreemptionDisabled(qName, preemptionDisabled); + preemptionDisabled = conf.getPreemptionDisabled(new QueuePath(qName), preemptionDisabled); qName += "."; } return preemptionDisabled; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java index 96e91d5945b7e..2ddb0d8ae1c51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor; import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Before; import org.junit.Test; @@ -366,7 +367,7 @@ public void testNodePartitionDisablePreemptionForSingleLevelQueue() "c\t" // app3 in c + "(1,1,n1,x,20,false)"; // 20x in n1 - conf.setPreemptionDisabled("root.b", true); + conf.setPreemptionDisabled(new QueuePath("root.b"), true); buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig); policy.editSchedule(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java index 2fb69f555728e..a11153a0482f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java @@ -20,6 +20,7 @@ import java.io.IOException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.junit.Before; import org.junit.Test; @@ -120,10 +121,15 @@ public void testNoIntraQueuePreemptionWithPreemptionDisabledOnQueues() * purpose is to test that disabling preemption on a specific queue will * avoid intra-queue preemption. */ - conf.setPreemptionDisabled("root.a", true); - conf.setPreemptionDisabled("root.b", true); - conf.setPreemptionDisabled("root.c", true); - conf.setPreemptionDisabled("root.d", true); + QueuePath a = new QueuePath("root.a"); + QueuePath b = new QueuePath("root.b"); + QueuePath c = new QueuePath("root.c"); + QueuePath d = new QueuePath("root.d"); + + conf.setPreemptionDisabled(a, true); + conf.setPreemptionDisabled(b, true); + conf.setPreemptionDisabled(c, true); + conf.setPreemptionDisabled(d, true); String labelsConfig = "=100,true;"; String nodesConfig = // n1 has no label diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockQueueHierarchy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockQueueHierarchy.java index ae4ff5a663ed2..44075b322d3aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockQueueHierarchy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockQueueHierarchy.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy; @@ -273,7 +274,7 @@ private void setupQueue(CSQueue queue, String q, String[] queueExprArray, // Setup preemption disabled when(queue.getPreemptionDisabled()).thenReturn( - conf.getPreemptionDisabled(queuePath, false)); + conf.getPreemptionDisabled(new QueuePath(queuePath), false)); // Setup other queue configurations Map otherConfigs = getOtherConfigurations( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index 0400f7dfcbf0e..a4444b4d479b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; @@ -667,10 +668,11 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a" }); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + final QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + conf.setQueues(root, new String[] { "a" }); + conf.setCapacityByLabel(root, "x", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final QueuePath A = QueuePath.addQueueToPath(root, "a"); conf.setCapacity(A, 100); conf.setAccessibleNodeLabels(A, ImmutableSet.of("x")); conf.setCapacityByLabel(A, "x", 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index cb28fc719fa8c..c8a034c545bdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -76,6 +76,18 @@ public class ReservationSystemTestUtil { public final static String RESERVATION_Q_SHORT = "dedicated"; public final static String reservationQ = "root." + RESERVATION_Q_SHORT; + public final static String DEDICATED_PATH = CapacitySchedulerConfiguration.ROOT + + CapacitySchedulerConfiguration.DOT + RESERVATION_Q_SHORT; + private final static String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default"; + private final static String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + private final static String A1_PATH = A_PATH + ".a1"; + private final static String A2_PATH = A_PATH + ".a2"; + private final static QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private final static QueuePath DEDICATED = new QueuePath(DEDICATED_PATH); + private final static QueuePath DEFAULT = new QueuePath(DEFAULT_PATH); + private final static QueuePath A = new QueuePath(A_PATH); + private final static QueuePath A1 = new QueuePath(A1_PATH); + private final static QueuePath A2 = new QueuePath(A2_PATH); public static ReservationId getNewReservationId() { return ReservationId.newInstance(rand.nextLong(), rand.nextLong()); @@ -293,25 +305,19 @@ public Resource answer(InvocationOnMock invocation) throws Throwable { public static void setupQueueConfiguration( CapacitySchedulerConfiguration conf) { // Define default queue - final String defQ = CapacitySchedulerConfiguration.ROOT + ".default"; + final String defQPath = CapacitySchedulerConfiguration.ROOT + ".default"; + final QueuePath defQ = new QueuePath(defQPath); conf.setCapacity(defQ, 10); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] {"default", "a", RESERVATION_Q_SHORT}); - - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); - - final String dedicated = CapacitySchedulerConfiguration.ROOT - + CapacitySchedulerConfiguration.DOT + RESERVATION_Q_SHORT; - conf.setCapacity(dedicated, 80); + conf.setCapacity(DEDICATED, 80); // Set as reservation queue - conf.setReservable(dedicated, true); + conf.setReservable(DEDICATED.getFullPath(), true); // Define 2nd-level queues - final String A1 = A + ".a1"; - final String A2 = A + ".a2"; conf.setQueues(A, new String[] {"a1", "a2"}); conf.setCapacity(A1, 30); conf.setCapacity(A2, 70); @@ -320,13 +326,11 @@ public static void setupQueueConfiguration( public static void setupDynamicQueueConfiguration( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] {RESERVATION_Q_SHORT}); - final String dedicated = CapacitySchedulerConfiguration.ROOT - + CapacitySchedulerConfiguration.DOT + RESERVATION_Q_SHORT; - conf.setCapacity(dedicated, 100); + conf.setCapacity(DEDICATED, 100); // Set as reservation queue - conf.setReservable(dedicated, true); + conf.setReservable(DEDICATED.getFullPath(), true); } public static String getFullReservationQueueName() { @@ -346,29 +350,23 @@ public static void updateQueueConfiguration( CapacitySchedulerConfiguration conf, String newQ) { // Define default queue final String prefix = CapacitySchedulerConfiguration.ROOT - + CapacitySchedulerConfiguration.DOT; - final String defQ = prefix + "default"; - conf.setCapacity(defQ, 5); + + CapacitySchedulerConfiguration.DOT; + conf.setCapacity(DEFAULT, 5); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] {"default", "a", RESERVATION_Q_SHORT, newQ}); - - final String A = prefix + "a"; conf.setCapacity(A, 5); - - final String dedicated = prefix + RESERVATION_Q_SHORT; - conf.setCapacity(dedicated, 10); + conf.setCapacity(DEDICATED, 10); // Set as reservation queue - conf.setReservable(dedicated, true); + conf.setReservable(DEDICATED.getFullPath(), true); - conf.setCapacity(prefix + newQ, 80); + final QueuePath newQueue = new QueuePath(prefix + newQ); + conf.setCapacity(newQueue, 80); // Set as reservation queue conf.setReservable(prefix + newQ, true); // Define 2nd-level queues - final String A1 = A + ".a1"; - final String A2 = A + ".a2"; conf.setQueues(A, new String[] { "a1", "a2" }); conf.setCapacity(A1, 30); conf.setCapacity(A2, 70); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java index c972c6ce7941f..20aa5511f8222 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java @@ -132,9 +132,9 @@ private void setupPlanFollower() throws Exception { QueuePath reservationQueuePath = ReservationSystemTestUtil.getFullReservationQueuePath(); CapacitySchedulerConfiguration csConf = cs.getConfiguration(); - csConf.setReservationWindow(reservationQ, 20L); + csConf.setReservationWindow(reservationQueuePath, 20L); csConf.setMaximumCapacity(reservationQueuePath, 40); - csConf.setAverageCapacity(reservationQ, 20); + csConf.setAverageCapacity(reservationQueuePath, 20); policy.init(reservationQ, csConf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java index 2e003151e79a2..0f1919b93baa2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.util.Times; import org.slf4j.event.Level; @@ -75,6 +76,10 @@ @RunWith(Parameterized.class) public class TestApplicationLifetimeMonitor { private final long maxLifetime = 30L; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final String CQ1 = "child1"; + private static final QueuePath CQ1_QUEUE_PATH = QueuePath.addQueueToPath(ROOT, CQ1); + private static final QueuePath DEFAULT_QUEUE_PATH = QueuePath.addQueueToPath(ROOT, "default"); private YarnConfiguration conf; @@ -390,7 +395,6 @@ public synchronized void updateApplicationStateInternal( } } - static final String CQ1 = "child1"; @Test(timeout = 120000) public void testInheritAppLifetimeFromParentQueue() throws Exception { YarnConfiguration yarnConf = conf; @@ -399,13 +403,10 @@ public void testInheritAppLifetimeFromParentQueue() throws Exception { if (scheduler.equals(CapacityScheduler.class)) { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {CQ1}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + CQ1, 100); - csConf.setMaximumLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT, maxRootLifetime); - csConf.setDefaultLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT, defaultRootLifetime); + csConf.setQueues(ROOT, new String[] {CQ1}); + csConf.setCapacity(CQ1_QUEUE_PATH, 100); + csConf.setMaximumLifetimePerQueue(ROOT, maxRootLifetime); + csConf.setDefaultLifetimePerQueue(ROOT, defaultRootLifetime); yarnConf = new YarnConfiguration(csConf); } @@ -465,17 +466,12 @@ public void testOverrideParentQueueMaxAppLifetime() throws Exception { if (scheduler.equals(CapacityScheduler.class)) { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {CQ1}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + CQ1, 100); - csConf.setMaximumLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT, maxRootLifetime); - csConf.setMaximumLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT + "." + CQ1, maxChildLifetime); - csConf.setDefaultLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT, defaultRootLifetime); - csConf.setDefaultLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT + "." + CQ1, maxChildLifetime); + csConf.setQueues(ROOT, new String[] {CQ1}); + csConf.setCapacity(CQ1_QUEUE_PATH, 100); + csConf.setMaximumLifetimePerQueue(ROOT, maxRootLifetime); + csConf.setMaximumLifetimePerQueue(CQ1_QUEUE_PATH, maxChildLifetime); + csConf.setDefaultLifetimePerQueue(ROOT, defaultRootLifetime); + csConf.setDefaultLifetimePerQueue(CQ1_QUEUE_PATH, maxChildLifetime); yarnConf = new YarnConfiguration(csConf); } @@ -529,16 +525,11 @@ public void testOverrideParentQueueDefaultAppLifetime() throws Exception { if (scheduler.equals(CapacityScheduler.class)) { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {CQ1}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + CQ1, 100); - csConf.setMaximumLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT, maxRootLifetime); - csConf.setMaximumLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT + "." + CQ1, maxChildLifetime); - csConf.setDefaultLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT + "." + CQ1, - defaultChildLifetime); + csConf.setQueues(ROOT, new String[] {CQ1}); + csConf.setCapacity(CQ1_QUEUE_PATH, 100); + csConf.setMaximumLifetimePerQueue(ROOT, maxRootLifetime); + csConf.setMaximumLifetimePerQueue(CQ1_QUEUE_PATH, maxChildLifetime); + csConf.setDefaultLifetimePerQueue(CQ1_QUEUE_PATH, defaultChildLifetime); yarnConf = new YarnConfiguration(csConf); } @@ -592,13 +583,11 @@ private CapacitySchedulerConfiguration setUpCSQueue(long maxLifetime, long defaultLifetime) { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] {"default"}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".default", 100); - csConf.setMaximumLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT + ".default", maxLifetime); - csConf.setDefaultLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT + ".default", defaultLifetime); + csConf.setCapacity(DEFAULT_QUEUE_PATH, 100); + csConf.setMaximumLifetimePerQueue(DEFAULT_QUEUE_PATH, maxLifetime); + csConf.setDefaultLifetimePerQueue(DEFAULT_QUEUE_PATH, defaultLifetime); return csConf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java index 1f1ae5c752186..033e060b972f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java @@ -27,47 +27,43 @@ public final class CapacitySchedulerQueueHelpers { - public static final String DEFAULT = CapacitySchedulerConfiguration.ROOT + ".default"; - public static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - public static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; - public static final String C = CapacitySchedulerConfiguration.ROOT + ".c"; - public static final String A_CHILD = A + ".a"; - public static final String A1 = A + ".a1"; - public static final String A2 = A + ".a2"; - public static final String A3 = A + ".a3"; - public static final String B1 = B + ".b1"; - public static final String B2 = B + ".b2"; - public static final String B3 = B + ".b3"; - public static final String C1 = C + ".c1"; - public static final String C2 = C + ".c2"; - public static final QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); - public static final QueuePath DEFAULT_QUEUE_PATH = new QueuePath(DEFAULT); - public static final QueuePath A_QUEUE_PATH = new QueuePath(A); - public static final QueuePath A_CHILD_QUEUE_PATH = new QueuePath(A_CHILD); - public static final QueuePath A1_QUEUE_PATH = new QueuePath(A1); - public static final QueuePath A2_QUEUE_PATH = new QueuePath(A2); - public static final QueuePath A3_QUEUE_PATH = new QueuePath(A3); - public static final QueuePath B_QUEUE_PATH = new QueuePath(B); - public static final QueuePath B1_QUEUE_PATH = new QueuePath(B1); - public static final QueuePath B2_QUEUE_PATH = new QueuePath(B2); - public static final QueuePath B3_QUEUE_PATH = new QueuePath(B3); - public static final QueuePath C_QUEUE_PATH = new QueuePath(C); - public static final QueuePath C1_QUEUE_PATH = new QueuePath(C1); - public static final QueuePath C2_QUEUE_PATH = new QueuePath(C2); + public static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default"; + public static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + public static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + public static final String A_CHILD_PATH = A_PATH + ".a"; + public static final String A1_PATH = A_PATH + ".a1"; + public static final String A2_PATH = A_PATH + ".a2"; + public static final String A3_PATH = A_PATH + ".a3"; + public static final String B1_PATH = B_PATH + ".b1"; + public static final String B2_PATH = B_PATH + ".b2"; + public static final String B3_PATH = B_PATH + ".b3"; + public static final String A1_B1_PATH = A1_PATH + ".b1"; + public static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + public static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH); + public static final QueuePath A = new QueuePath(A_PATH); + public static final QueuePath A_CHILD = new QueuePath(A_CHILD_PATH); + public static final QueuePath A1 = new QueuePath(A1_PATH); + public static final QueuePath A2 = new QueuePath(A2_PATH); + public static final QueuePath A3 = new QueuePath(A3_PATH); + public static final QueuePath B = new QueuePath(B_PATH); + public static final QueuePath B1 = new QueuePath(B1_PATH); + public static final QueuePath B2 = new QueuePath(B2_PATH); + public static final QueuePath B3 = new QueuePath(B3_PATH); + public static final QueuePath A1_B1 = new QueuePath(A1_B1_PATH); public static final float A_CAPACITY = 10.5f; public static final float B_CAPACITY = 89.5f; - public static final String P1 = CapacitySchedulerConfiguration.ROOT + ".p1"; - public static final String P2 = CapacitySchedulerConfiguration.ROOT + ".p2"; - public static final String X1 = P1 + ".x1"; - public static final String X2 = P1 + ".x2"; - public static final String Y1 = P2 + ".y1"; - public static final String Y2 = P2 + ".y2"; - public static final QueuePath P1_QUEUE_PATH = new QueuePath(P1); - public static final QueuePath P2_QUEUE_PATH = new QueuePath(P2); - public static final QueuePath X1_QUEUE_PATH = new QueuePath(X1); - public static final QueuePath X2_QUEUE_PATH = new QueuePath(X2); - public static final QueuePath Y1_QUEUE_PATH = new QueuePath(Y1); - public static final QueuePath Y2_QUEUE_PATH = new QueuePath(Y2); + public static final String P1_PATH = CapacitySchedulerConfiguration.ROOT + ".p1"; + public static final String P2_PATH = CapacitySchedulerConfiguration.ROOT + ".p2"; + public static final String X1_PATH = P1_PATH + ".x1"; + public static final String X2_PATH = P1_PATH + ".x2"; + public static final String Y1_PATH = P2_PATH + ".y1"; + public static final String Y2_PATH = P2_PATH + ".y2"; + public static final QueuePath P1 = new QueuePath(P1_PATH); + public static final QueuePath P2 = new QueuePath(P2_PATH); + public static final QueuePath X1 = new QueuePath(X1_PATH); + public static final QueuePath X2 = new QueuePath(X2_PATH); + public static final QueuePath Y1 = new QueuePath(Y1_PATH); + public static final QueuePath Y2 = new QueuePath(Y2_PATH); public static final float A1_CAPACITY = 30; public static final float A2_CAPACITY = 70; public static final float B1_CAPACITY = 79.2f; @@ -92,24 +88,24 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"}); + conf.setQueues(ROOT, new String[]{"a", "b"}); - conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); - conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); + conf.setCapacity(A, A_CAPACITY); + conf.setCapacity(B, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); + conf.setCapacity(A1, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); + conf.setCapacity(A2, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); conf.setQueues(B, new String[]{"b1", "b2", "b3"}); - conf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); + conf.setCapacity(B1, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); - conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY); + conf.setCapacity(B2, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); + conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); return conf; @@ -119,26 +115,26 @@ public static CapacitySchedulerConfiguration setupAdditionalQueues( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"}); + conf.setQueues(ROOT, new String[]{"a", "b"}); - conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); - conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); + conf.setCapacity(A, A_CAPACITY); + conf.setCapacity(B, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2", "a3"}); - conf.setCapacity(A1_QUEUE_PATH, 30.0f); + conf.setCapacity(A1, 30.0f); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2_QUEUE_PATH, 30.0f); + conf.setCapacity(A2, 30.0f); conf.setUserLimitFactor(A2, 100.0f); - conf.setCapacity(A3_QUEUE_PATH, 40.0f); - conf.setUserLimitFactor("root.a.a3", 100.0f); + conf.setCapacity(A3, 40.0f); + conf.setUserLimitFactor(A3, 100.0f); conf.setQueues(B, new String[]{"b1", "b2", "b3"}); - conf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); + conf.setCapacity(B1, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); - conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY); + conf.setCapacity(B2, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); + conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); return conf; @@ -157,17 +153,17 @@ public static CapacitySchedulerConfiguration setupQueueConfAmbiguousQueue( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[]{"a", "b"}); - conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); - conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); + conf.setCapacity(A, A_CAPACITY); + conf.setCapacity(B, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a", "a1"}); - conf.setCapacity(A_CHILD_QUEUE_PATH, A1_CAPACITY); + conf.setCapacity(A_CHILD, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A1_QUEUE_PATH, A2_CAPACITY); + conf.setCapacity(A1, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); return conf; @@ -186,17 +182,17 @@ public static CapacitySchedulerConfiguration setupQueueConfWithoutChildrenOfB( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[]{"a", "b"}); - conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); - conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); + conf.setCapacity(A, A_CAPACITY); + conf.setCapacity(B, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); + conf.setCapacity(A1, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); + conf.setCapacity(A2, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); return conf; @@ -215,23 +211,23 @@ public static CapacitySchedulerConfiguration setupQueueConfigurationWithoutB1( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[]{"a", "b"}); - conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); - conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); + conf.setCapacity(A, A_CAPACITY); + conf.setCapacity(B, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); + conf.setCapacity(A1, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); + conf.setCapacity(A2, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); conf.setQueues(B, new String[]{"b2", "b3"}); - conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY + B1_CAPACITY); //as B1 is deleted + conf.setCapacity(B2, B2_CAPACITY + B1_CAPACITY); //as B1 is deleted conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); + conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); return conf; @@ -252,31 +248,32 @@ public static CapacitySchedulerConfiguration setupQueueConfigurationWithB1AsPare CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[]{"a", "b"}); - conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); - conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); + conf.setCapacity(A, A_CAPACITY); + conf.setCapacity(B, B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); + conf.setCapacity(A1, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); + conf.setCapacity(A2, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); conf.setQueues(B, new String[]{"b1", "b2", "b3"}); - conf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); + conf.setCapacity(B1, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); - conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY); + conf.setCapacity(B2, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); + conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); // Set childQueue for B1 conf.setQueues(B1, new String[]{"b11"}); - final String b11 = B1 + ".b11"; - conf.setCapacity(new QueuePath(b11), 100.0f); + final String b11Path = B1 + ".b11"; + final QueuePath b11 = new QueuePath(b11Path); + conf.setCapacity(b11, 100.0f); conf.setUserLimitFactor(b11, 100.0f); return conf; @@ -290,15 +287,15 @@ public static CapacitySchedulerConfiguration setupQueueConfigurationWithoutB( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a"}); + conf.setQueues(ROOT, new String[]{"a"}); - conf.setCapacity(A_QUEUE_PATH, A_CAPACITY + B_CAPACITY); + conf.setCapacity(A, A_CAPACITY + B_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[]{"a1", "a2"}); - conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); + conf.setCapacity(A1, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); + conf.setCapacity(A2, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); return conf; @@ -308,15 +305,15 @@ public static CapacitySchedulerConfiguration setupBlockedQueueConfiguration( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[]{"a", "b"}); - conf.setCapacity(A_QUEUE_PATH, 80f); - conf.setCapacity(B_QUEUE_PATH, 20f); + conf.setCapacity(A, 80f); + conf.setCapacity(B, 20f); conf.setUserLimitFactor(A, 100); conf.setUserLimitFactor(B, 100); - conf.setMaximumCapacity(A_QUEUE_PATH, 100); - conf.setMaximumCapacity(B_QUEUE_PATH, 100); + conf.setMaximumCapacity(A, 100); + conf.setMaximumCapacity(B, 100); return conf; } @@ -324,26 +321,26 @@ public static CapacitySchedulerConfiguration setupOtherBlockedQueueConfiguration CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[]{"p1", "p2"}); - conf.setCapacity(P1_QUEUE_PATH, 50f); - conf.setMaximumCapacity(P1_QUEUE_PATH, 50f); - conf.setCapacity(P2_QUEUE_PATH, 50f); - conf.setMaximumCapacity(P2_QUEUE_PATH, 100f); + conf.setCapacity(P1, 50f); + conf.setMaximumCapacity(P1, 50f); + conf.setCapacity(P2, 50f); + conf.setMaximumCapacity(P2, 100f); // Define 2nd-level queues conf.setQueues(P1, new String[]{"x1", "x2"}); - conf.setCapacity(X1_QUEUE_PATH, 80f); - conf.setMaximumCapacity(X1_QUEUE_PATH, 100f); + conf.setCapacity(X1, 80f); + conf.setMaximumCapacity(X1, 100f); conf.setUserLimitFactor(X1, 2f); - conf.setCapacity(X2_QUEUE_PATH, 20f); - conf.setMaximumCapacity(X2_QUEUE_PATH, 100f); + conf.setCapacity(X2, 20f); + conf.setMaximumCapacity(X2, 100f); conf.setUserLimitFactor(X2, 2f); conf.setQueues(P2, new String[]{"y1", "y2"}); - conf.setCapacity(Y1_QUEUE_PATH, 80f); + conf.setCapacity(Y1, 80f); conf.setUserLimitFactor(Y1, 2f); - conf.setCapacity(Y2_QUEUE_PATH, 20f); + conf.setCapacity(Y2, 20f); conf.setUserLimitFactor(Y2, 2f); return conf; } @@ -368,13 +365,13 @@ public float getAbsCapacity() { public static Map getDefaultCapacities(float capA, float capB) { Map capacities = new HashMap<>(); - capacities.put(A, new ExpectedCapacities(capA, 1.0f)); - capacities.put(B, new ExpectedCapacities(capB, 1.0f)); - capacities.put(A1, new ExpectedCapacities((A1_CAPACITY / 100.0f), capA)); - capacities.put(A2, new ExpectedCapacities((A2_CAPACITY / 100.0f), capA)); - capacities.put(B1, new ExpectedCapacities((B1_CAPACITY / 100.0f), capB)); - capacities.put(B2, new ExpectedCapacities((B2_CAPACITY / 100.0f), capB)); - capacities.put(B3, new ExpectedCapacities((B3_CAPACITY / 100.0f), capB)); + capacities.put(A.getFullPath(), new ExpectedCapacities(capA, 1.0f)); + capacities.put(B.getFullPath(), new ExpectedCapacities(capB, 1.0f)); + capacities.put(A1.getFullPath(), new ExpectedCapacities((A1_CAPACITY / 100.0f), capA)); + capacities.put(A2.getFullPath(), new ExpectedCapacities((A2_CAPACITY / 100.0f), capA)); + capacities.put(B1.getFullPath(), new ExpectedCapacities((B1_CAPACITY / 100.0f), capB)); + capacities.put(B2.getFullPath(), new ExpectedCapacities((B2_CAPACITY / 100.0f), capB)); + capacities.put(B3.getFullPath(), new ExpectedCapacities((B3_CAPACITY / 100.0f), capB)); return capacities; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index 8d68cbf7932eb..8da88e9e427cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -48,6 +48,8 @@ public class TestAbsoluteResourceConfiguration { private static final String QUEUEA2 = "queueA2"; private static final String QUEUEB1 = "queueB1"; + private static final QueuePath ROOT = + new QueuePath(CapacitySchedulerConfiguration.ROOT); private static final QueuePath QUEUEA_FULL = new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUEA); private static final QueuePath QUEUEB_FULL = @@ -103,9 +105,9 @@ public class TestAbsoluteResourceConfiguration { private CapacitySchedulerConfiguration setupNormalizationConfiguration() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[]{QUEUEA, QUEUEB}); - csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[]{QUEUEA1, QUEUEA2}); + csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2}); // 60, 28 csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resource.newInstance(50 * GB, 20)); @@ -119,23 +121,23 @@ private CapacitySchedulerConfiguration setupNormalizationConfiguration() { private CapacitySchedulerConfiguration setupSimpleQueueConfiguration( boolean isCapacityNeeded) { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[]{QUEUEA, QUEUEB, QUEUEC, QUEUED}); // Set default capacities like normal configuration. if (isCapacityNeeded) { - csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f); - csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f); - csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f); - csConf.setCapacity(QUEUED_FULL.getFullPath(), 25f); + csConf.setCapacity(QUEUEA_FULL, 50f); + csConf.setCapacity(QUEUEB_FULL, 25f); + csConf.setCapacity(QUEUEC_FULL, 25f); + csConf.setCapacity(QUEUED_FULL, 25f); } - csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL.getFullPath(), true); + csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL, true); // Setup leaf queue template configs - csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL.getFullPath(), "", + csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL, "", QUEUE_D_TEMPL_MINRES); - csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL.getFullPath(), "", + csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL, "", QUEUE_D_TEMPL_MAXRES); return csConf; @@ -144,19 +146,19 @@ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration( private CapacitySchedulerConfiguration setupComplexQueueConfiguration( boolean isCapacityNeeded) { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[]{QUEUEA, QUEUEB, QUEUEC}); - csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[]{QUEUEA1, QUEUEA2}); - csConf.setQueues(QUEUEB_FULL.getFullPath(), new String[]{QUEUEB1}); + csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2}); + csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1}); // Set default capacities like normal configuration. if (isCapacityNeeded) { - csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f); - csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f); - csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f); - csConf.setCapacity(QUEUEA1_FULL.getFullPath(), 50f); - csConf.setCapacity(QUEUEA2_FULL.getFullPath(), 50f); - csConf.setCapacity(QUEUEB1_FULL.getFullPath(), 100f); + csConf.setCapacity(QUEUEA_FULL, 50f); + csConf.setCapacity(QUEUEB_FULL, 25f); + csConf.setCapacity(QUEUEC_FULL, 25f); + csConf.setCapacity(QUEUEA1_FULL, 50f); + csConf.setCapacity(QUEUEA2_FULL, 50f); + csConf.setCapacity(QUEUEB1_FULL, 100f); } return csConf; @@ -225,22 +227,22 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() Assert.assertEquals("Min resource configured for QUEUEA is not correct", QUEUE_A_MINRES, - csConf.getMinimumResourceRequirement("", QUEUEA_FULL.getFullPath(), resourceTypes)); + csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes)); Assert.assertEquals("Max resource configured for QUEUEA is not correct", QUEUE_A_MAXRES, - csConf.getMaximumResourceRequirement("", QUEUEA_FULL.getFullPath(), resourceTypes)); + csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes)); Assert.assertEquals("Min resource configured for QUEUEB is not correct", QUEUE_B_MINRES, - csConf.getMinimumResourceRequirement("", QUEUEB_FULL.getFullPath(), resourceTypes)); + csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes)); Assert.assertEquals("Max resource configured for QUEUEB is not correct", QUEUE_B_MAXRES, - csConf.getMaximumResourceRequirement("", QUEUEB_FULL.getFullPath(), resourceTypes)); + csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes)); Assert.assertEquals("Min resource configured for QUEUEC is not correct", QUEUE_C_MINRES, - csConf.getMinimumResourceRequirement("", QUEUEC_FULL.getFullPath(), resourceTypes)); + csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes)); Assert.assertEquals("Max resource configured for QUEUEC is not correct", QUEUE_C_MAXRES, - csConf.getMaximumResourceRequirement("", QUEUEC_FULL.getFullPath(), resourceTypes)); + csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes)); csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -563,9 +565,9 @@ public void testComplexValidateAbsoluteResourceConfig() throws Exception { // 1. Explicitly set percentage based config for parent queues. This will // make Queue A,B and C with percentage based and A1,A2 or B1 with absolute // resource. - csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f); - csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f); - csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f); + csConf.setCapacity(QUEUEA_FULL, 50f); + csConf.setCapacity(QUEUEB_FULL, 25f); + csConf.setCapacity(QUEUEC_FULL, 25f); // Get queue object to verify min/max resource configuration. CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); @@ -608,15 +610,15 @@ public void testValidateAbsoluteResourceConfig() throws Exception { // create conf with basic queue configuration. CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] {QUEUEA, QUEUEB}); - csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[] {QUEUEA1, QUEUEA2}); + csConf.setQueues(QUEUEA_FULL, new String[] {QUEUEA1, QUEUEA2}); // Set default capacities like normal configuration. - csConf.setCapacity(QUEUEA_FULL.getFullPath(), "[memory=125]"); - csConf.setCapacity(QUEUEB_FULL.getFullPath(), "[memory=0]"); - csConf.setCapacity(QUEUEA1_FULL.getFullPath(), "[memory=100]"); - csConf.setCapacity(QUEUEA2_FULL.getFullPath(), "[memory=25]"); + csConf.setCapacity(QUEUEA_FULL, "[memory=125]"); + csConf.setCapacity(QUEUEB_FULL, "[memory=0]"); + csConf.setCapacity(QUEUEA1_FULL, "[memory=100]"); + csConf.setCapacity(QUEUEA2_FULL, "[memory=25]"); // Update min/max resource to queueA csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES); @@ -636,8 +638,8 @@ public void testValidateAbsoluteResourceConfig() throws Exception { // doesnt throw exception saying "Parent queue 'root.A' and // child queue 'root.A.A2' should use either percentage // based capacityconfiguration or absolute resource together for label" - csConf.setCapacity(QUEUEA1_FULL.getFullPath(), "[memory=125]"); - csConf.setCapacity(QUEUEA2_FULL.getFullPath(), "[memory=0]"); + csConf.setCapacity(QUEUEA1_FULL, "[memory=125]"); + csConf.setCapacity(QUEUEA2_FULL, "[memory=0]"); // Get queue object to verify min/max resource configuration. CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java index e826bcb8fbdb4..0fd240992ebaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java @@ -70,6 +70,13 @@ public class TestAbsoluteResourceWithAutoQueue private static final String QUEUED_FULL = CapacitySchedulerConfiguration.ROOT + "." + QUEUED; + private static final QueuePath ROOT_QUEUE_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A_QUEUE_PATH = new QueuePath(QUEUEA_FULL); + private static final QueuePath B_QUEUE_PATH = new QueuePath(QUEUEB_FULL); + private static final QueuePath C_QUEUE_PATH = new QueuePath(QUEUEC_FULL); + private static final QueuePath D_QUEUE_PATH = new QueuePath(QUEUED_FULL); + private static final Resource QUEUE_A_MINRES = Resource.newInstance(100 * GB, 10); private static final Resource QUEUE_A_MAXRES = @@ -96,15 +103,15 @@ public void setUp() throws Exception { private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration( CapacitySchedulerConfiguration csConf) { // Update min/max resource to queueA/B/C - csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEA_FULL), QUEUE_A_MINRES); - csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEB_FULL), QUEUE_B_MINRES); - csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEC_FULL), QUEUE_C_MINRES); - csConf.setMinimumResourceRequirement("", new QueuePath(QUEUED_FULL), QUEUE_D_MINRES); + csConf.setMinimumResourceRequirement("", A_QUEUE_PATH, QUEUE_A_MINRES); + csConf.setMinimumResourceRequirement("", B_QUEUE_PATH, QUEUE_B_MINRES); + csConf.setMinimumResourceRequirement("", C_QUEUE_PATH, QUEUE_C_MINRES); + csConf.setMinimumResourceRequirement("", D_QUEUE_PATH, QUEUE_D_MINRES); - csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEA_FULL), QUEUE_A_MAXRES); - csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEB_FULL), QUEUE_B_MAXRES); - csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEC_FULL), QUEUE_C_MAXRES); - csConf.setMaximumResourceRequirement("", new QueuePath(QUEUED_FULL), QUEUE_D_MAXRES); + csConf.setMaximumResourceRequirement("", A_QUEUE_PATH, QUEUE_A_MAXRES); + csConf.setMaximumResourceRequirement("", B_QUEUE_PATH, QUEUE_B_MAXRES); + csConf.setMaximumResourceRequirement("", C_QUEUE_PATH, QUEUE_C_MAXRES); + csConf.setMaximumResourceRequirement("", D_QUEUE_PATH, QUEUE_D_MAXRES); return csConf; } @@ -119,30 +126,30 @@ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration( boolean isCapacityNeeded) { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT_QUEUE_PATH, new String[] { QUEUEA, QUEUEB, QUEUEC, QUEUED }); // Set default capacities like normal configuration. if (isCapacityNeeded) { - csConf.setCapacity(QUEUEA_FULL, 50f); - csConf.setCapacity(QUEUEB_FULL, 25f); - csConf.setCapacity(QUEUEC_FULL, 25f); - csConf.setCapacity(QUEUED_FULL, 25f); + csConf.setCapacity(A_QUEUE_PATH, 50f); + csConf.setCapacity(B_QUEUE_PATH, 25f); + csConf.setCapacity(C_QUEUE_PATH, 25f); + csConf.setCapacity(D_QUEUE_PATH, 25f); } - csConf.setAutoCreateChildQueueEnabled(QUEUEC_FULL, true); + csConf.setAutoCreateChildQueueEnabled(C_QUEUE_PATH, true); - csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUEC_FULL, "", + csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(C_QUEUE_PATH, "", QUEUE_C_MINRES); - csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUEC_FULL, "", + csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(C_QUEUE_PATH, "", QUEUE_C_MAXRES); - csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL, true); + csConf.setAutoCreateChildQueueEnabled(D_QUEUE_PATH, true); // Setup leaf queue template configs - csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL, "", + csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(D_QUEUE_PATH, "", Resource.newInstance(10 * GB, 2)); - csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL, "", + csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(D_QUEUE_PATH, "", QUEUE_D_MAXRES); return csConf; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index a2f7bac18b377..10417f8a61e47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -141,22 +141,35 @@ public void setUp() throws IOException { doReturn(100).when(queue).getMaxApplications(); doReturn(25).when(queue).getMaxApplicationsPerUser(); } - + private static final String A = "a"; private static final String B = "b"; + private static final String C = "c"; + private static final String D = "d"; + private static final String AA1 = "a1"; + private static final String AA2 = "a2"; + private static final String AA3 = "a3"; + private static final QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT, A); + private static final QueuePath B_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT, B); + private static final QueuePath C_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT, C); + private static final QueuePath D_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT, D); + private static final QueuePath AA1_QUEUE_PATH = QueuePath.createFromQueues( + CapacitySchedulerConfiguration.ROOT, A, AA1); + private static final QueuePath AA2_QUEUE_PATH = QueuePath.createFromQueues( + CapacitySchedulerConfiguration.ROOT, A, AA2); + private static final QueuePath AA3_QUEUE_PATH = QueuePath.createFromQueues( + CapacitySchedulerConfiguration.ROOT, A, AA3); private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B}); + conf.setQueues(ROOT_QUEUE_PATH, new String[] {A, B}); - final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; - conf.setCapacity(Q_A, 10); - - final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; - conf.setCapacity(Q_B, 90); + conf.setCapacity(A_QUEUE_PATH, 10); + conf.setCapacity(B_QUEUE_PATH, 90); - conf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 50); - conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + "." + A, 5.0f); + conf.setUserLimit(A_QUEUE_PATH, 50); + conf.setUserLimitFactor(A_QUEUE_PATH, 5.0f); LOG.info("Setup top-level queues a and b"); } @@ -570,7 +583,7 @@ public void testActiveLimitsWithKilledApps() throws Exception { public void testHeadroom() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 25); + csConf.setUserLimit(A_QUEUE_PATH, 25); setupQueueConfiguration(csConf); // Say cluster has 100 nodes of 16G each @@ -715,53 +728,46 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config); // Define top-level - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT_QUEUE_PATH, new String[]{"a", "b", "c", "d"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); + conf.setCapacityByLabel(ROOT_QUEUE_PATH, "x", 100); + conf.setCapacityByLabel(ROOT_QUEUE_PATH, "y", 100); + conf.setCapacityByLabel(ROOT_QUEUE_PATH, "z", 100); conf.setInt(CapacitySchedulerConfiguration.QUEUE_GLOBAL_MAX_APPLICATION, 20); conf.setInt("yarn.scheduler.capacity.root.a.a1.maximum-applications", 1); conf.setFloat("yarn.scheduler.capacity.root.d.user-limit-factor", 0.1f); conf.setInt("yarn.scheduler.capacity.maximum-applications", 4); - final String a = CapacitySchedulerConfiguration.ROOT + ".a"; - final String b = CapacitySchedulerConfiguration.ROOT + ".b"; - final String c = CapacitySchedulerConfiguration.ROOT + ".c"; - final String d = CapacitySchedulerConfiguration.ROOT + ".d"; - final String aa1 = a + ".a1"; - final String aa2 = a + ".a2"; - final String aa3 = a + ".a3"; - - conf.setQueues(a, new String[]{"a1", "a2", "a3"}); - conf.setCapacity(a, 50); - conf.setCapacity(b, 50); - conf.setCapacity(c, 0); - conf.setCapacity(d, 0); - conf.setCapacity(aa1, 50); - conf.setCapacity(aa2, 50); - conf.setCapacity(aa3, 0); - - conf.setCapacityByLabel(a, "y", 25); - conf.setCapacityByLabel(b, "y", 50); - conf.setCapacityByLabel(c, "y", 25); - conf.setCapacityByLabel(d, "y", 0); - - conf.setCapacityByLabel(a, "x", 50); - conf.setCapacityByLabel(b, "x", 50); - - conf.setCapacityByLabel(a, "z", 50); - conf.setCapacityByLabel(b, "z", 50); - - conf.setCapacityByLabel(aa1, "x", 100); - conf.setCapacityByLabel(aa2, "x", 0); - - conf.setCapacityByLabel(aa1, "y", 25); - conf.setCapacityByLabel(aa2, "y", 75); - - conf.setCapacityByLabel(aa2, "z", 75); - conf.setCapacityByLabel(aa3, "z", 25); + + conf.setQueues(A_QUEUE_PATH, new String[]{"a1", "a2", "a3"}); + conf.setCapacity(A_QUEUE_PATH, 50); + conf.setCapacity(B_QUEUE_PATH, 50); + conf.setCapacity(C_QUEUE_PATH, 0); + conf.setCapacity(D_QUEUE_PATH, 0); + conf.setCapacity(AA1_QUEUE_PATH, 50); + conf.setCapacity(AA2_QUEUE_PATH, 50); + conf.setCapacity(AA3_QUEUE_PATH, 0); + + conf.setCapacityByLabel(A_QUEUE_PATH, "y", 25); + conf.setCapacityByLabel(B_QUEUE_PATH, "y", 50); + conf.setCapacityByLabel(C_QUEUE_PATH, "y", 25); + conf.setCapacityByLabel(D_QUEUE_PATH, "y", 0); + + conf.setCapacityByLabel(A_QUEUE_PATH, "x", 50); + conf.setCapacityByLabel(B_QUEUE_PATH, "x", 50); + + conf.setCapacityByLabel(A_QUEUE_PATH, "z", 50); + conf.setCapacityByLabel(B_QUEUE_PATH, "z", 50); + + conf.setCapacityByLabel(AA1_QUEUE_PATH, "x", 100); + conf.setCapacityByLabel(AA2_QUEUE_PATH, "x", 0); + + conf.setCapacityByLabel(AA1_QUEUE_PATH, "y", 25); + conf.setCapacityByLabel(AA2_QUEUE_PATH, "y", 75); + + conf.setCapacityByLabel(AA2_QUEUE_PATH, "z", 75); + conf.setCapacityByLabel(AA3_QUEUE_PATH, "z", 25); return conf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index 0c78b427cc700..b65b95cf7b6dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -77,6 +77,15 @@ public class TestApplicationLimitsByPartition { final static int GB = 1024; + final static String A1_PATH = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; + final static String B1_PATH = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; + final static String B2_PATH = CapacitySchedulerConfiguration.ROOT + ".a" + ".b2"; + final static String C1_PATH = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1"; + final static QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + final static QueuePath A1 = new QueuePath(A1_PATH); + final static QueuePath B1 = new QueuePath(B1_PATH); + final static QueuePath B2 = new QueuePath(B2_PATH); + final static QueuePath C1 = new QueuePath(C1_PATH); LeafQueue queue; RMNodeLabelsManager mgr; @@ -136,11 +145,8 @@ public void testAMResourceLimitWithLabels() throws Exception { // After getting queue conf, configure AM resource percent for Queue A1 // as 0.2 (Label X) and for Queue C1 as 0.2 (Empty Label) - final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; - final String C1 = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1"; - final QueuePath C1_QUEUE_PATH = new QueuePath(C1); config.setMaximumAMResourcePercentPerPartition(A1, "x", 0.2f); - config.setMaximumApplicationMasterResourcePerQueuePercent(C1_QUEUE_PATH, 0.2f); + config.setMaximumApplicationMasterResourcePerQueuePercent(C1, 0.2f); // Now inject node label manager with this updated config MockRM rm1 = new MockRM(config) { @@ -293,11 +299,8 @@ public void testAtleastOneAMRunPerPartition() throws Exception { // After getting queue conf, configure AM resource percent for Queue A1 // as 0.15 (Label X) and for Queue C1 as 0.15 (Empty Label) - final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; - final String C1 = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1"; - final QueuePath C1_QUEUE_PATH = new QueuePath(C1); config.setMaximumAMResourcePercentPerPartition(A1, "x", 0.15f); - config.setMaximumApplicationMasterResourcePerQueuePercent(C1_QUEUE_PATH, 0.15f); + config.setMaximumApplicationMasterResourcePerQueuePercent(C1, 0.15f); // inject node label manager MockRM rm1 = new MockRM(config) { @Override @@ -403,9 +406,7 @@ public void testDefaultAMLimitFromQueueForPartition() throws Exception { // After getting queue conf, configure AM resource percent for Queue A1 // as 0.2 (not for partition, rather in queue level) - final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; - final QueuePath A1_QUEUE_PATH = new QueuePath(A1); - config.setMaximumApplicationMasterResourcePerQueuePercent(A1_QUEUE_PATH, 0.2f); + config.setMaximumApplicationMasterResourcePerQueuePercent(A1, 0.2f); // inject node label manager MockRM rm1 = new MockRM(config) { @Override @@ -486,7 +487,6 @@ public void testUserAMResourceLimitWithLabels() throws Exception { // as 0.4 (Label X). Also set userlimit as 50% for this queue. So when we // have two users submitting applications, each user will get 50% of AM // resource which is available in this partition. - final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; config.setMaximumAMResourcePercentPerPartition(A1, "x", 0.4f); config.setUserLimit(A1, 50); @@ -628,11 +628,8 @@ public void testAMResourceLimitForMultipleApplications() throws Exception { * A1 : 0.25 * B2 : 0.15 */ - final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; - final String B1 = CapacitySchedulerConfiguration.ROOT + ".b" + ".b1"; - final QueuePath B1_QUEUE_PATH = new QueuePath(B1); config.setMaximumAMResourcePercentPerPartition(A1, "y", 0.25f); - config.setMaximumApplicationMasterResourcePerQueuePercent(B1_QUEUE_PATH, 0.15f); + config.setMaximumApplicationMasterResourcePerQueuePercent(B1, 0.15f); // Now inject node label manager with this updated config MockRM rm1 = new MockRM(config) { @@ -760,8 +757,6 @@ public void testHeadroom() throws Exception { CapacitySchedulerConfiguration csConf = (CapacitySchedulerConfiguration) TestUtils .getComplexConfigurationWithQueueLabels(conf); - final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; - final String B2 = CapacitySchedulerConfiguration.ROOT + ".b" + ".b2"; csConf.setUserLimit(A1, 25); csConf.setUserLimit(B2, 25); @@ -945,9 +940,9 @@ public void testAMLimitByAllResources() throws Exception { new CapacitySchedulerConfiguration(); csconf.setResourceComparator(DominantResourceCalculator.class); String queueName = "a1"; - csconf.setQueues(CapacitySchedulerConfiguration.ROOT, + csconf.setQueues(ROOT, new String[] {queueName}); - csconf.setCapacity("root." + queueName, 100); + csconf.setCapacity(A1, 100); ResourceInformation res0 = ResourceInformation.newInstance("memory-mb", ResourceInformation.MEMORY_MB.getUnits(), GB, Long.MAX_VALUE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java index 598bd49b0e7bf..e2011ff9cdb98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java @@ -40,28 +40,36 @@ public class TestApplicationPriorityACLConfiguration { private static final String QUEUEB = "queueB"; private static final String QUEUEC = "queueC"; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A_QUEUE_PATH = new QueuePath( + CapacitySchedulerConfiguration.ROOT + "." + QUEUEA); + private static final QueuePath B_QUEUE_PATH = new QueuePath( + CapacitySchedulerConfiguration.ROOT + "." + QUEUEB); + private static final QueuePath C_QUEUE_PATH = new QueuePath( + CapacitySchedulerConfiguration.ROOT + "." + QUEUEC); + + @Test public void testSimpleACLConfiguration() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[]{QUEUEA, QUEUEB, QUEUEC}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 25f); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEC, 25f); + csConf.setCapacity(A_QUEUE_PATH, 50f); + csConf.setCapacity(B_QUEUE_PATH, 25f); + csConf.setCapacity(C_QUEUE_PATH, 25f); // Success case: Configure one user/group level priority acl for queue A. String[] aclsForA = new String[2]; aclsForA[0] = QUEUE_A_USER; aclsForA[1] = QUEUE_A_GROUP; - csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, + csConf.setPriorityAcls(A_QUEUE_PATH, Priority.newInstance(maxPriorityQueueA), Priority.newInstance(defaultPriorityQueueA), aclsForA); // Try to get the ACL configs and make sure there are errors/exceptions List pGroupA = csConf.getPriorityAcls( - CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, - Priority.newInstance(clusterMaxPriority)); + A_QUEUE_PATH, Priority.newInstance(clusterMaxPriority)); // Validate! verifyACLs(pGroupA, QUEUE_A_USER, QUEUE_A_GROUP, maxPriorityQueueA, @@ -71,34 +79,32 @@ public void testSimpleACLConfiguration() throws Exception { @Test public void testACLConfigurationForInvalidCases() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[]{QUEUEA, QUEUEB, QUEUEC}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 25f); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEC, 25f); + csConf.setCapacity(A_QUEUE_PATH, 50f); + csConf.setCapacity(B_QUEUE_PATH, 25f); + csConf.setCapacity(C_QUEUE_PATH, 25f); // Success case: Configure one user/group level priority acl for queue A. String[] aclsForA = new String[2]; aclsForA[0] = QUEUE_A_USER; aclsForA[1] = QUEUE_A_GROUP; - csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, + csConf.setPriorityAcls(A_QUEUE_PATH, Priority.newInstance(maxPriorityQueueA), Priority.newInstance(defaultPriorityQueueA), aclsForA); String[] aclsForB = new String[1]; aclsForB[0] = QUEUE_B_USER; - csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, + csConf.setPriorityAcls(B_QUEUE_PATH, Priority.newInstance(maxPriorityQueueB), Priority.newInstance(defaultPriorityQueueB), aclsForB); // Try to get the ACL configs and make sure there are errors/exceptions List pGroupA = csConf.getPriorityAcls( - CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, - Priority.newInstance(clusterMaxPriority)); + A_QUEUE_PATH, Priority.newInstance(clusterMaxPriority)); List pGroupB = csConf.getPriorityAcls( - CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, - Priority.newInstance(clusterMaxPriority)); + B_QUEUE_PATH, Priority.newInstance(clusterMaxPriority)); // Validate stored ACL values with configured ones. verifyACLs(pGroupA, QUEUE_A_USER, QUEUE_A_GROUP, maxPriorityQueueA, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java index cf9a01045b991..ab82aca001e3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java @@ -177,24 +177,24 @@ private void verifyAppPriorityIsAccepted(String submitter, @Override protected Configuration createConfiguration() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[]{QUEUEA, QUEUEB, QUEUEC}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 25f); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEC, 25f); + csConf.setCapacity(A_QUEUE_PATH, 50f); + csConf.setCapacity(B_QUEUE_PATH, 25f); + csConf.setCapacity(C_QUEUE_PATH, 25f); String[] aclsForA = new String[2]; aclsForA[0] = QUEUE_A_USER; aclsForA[1] = QUEUE_A_GROUP; - csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, + csConf.setPriorityAcls(A_QUEUE_PATH, Priority.newInstance(maxPriorityQueueA), Priority.newInstance(defaultPriorityQueueA), aclsForA); String[] aclsForB = new String[2]; aclsForB[0] = QUEUE_B_USER; aclsForB[1] = QUEUE_B_GROUP; - csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, + csConf.setPriorityAcls(B_QUEUE_PATH, Priority.newInstance(maxPriorityQueueB), Priority.newInstance(defaultPriorityQueueB), aclsForB); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index 896abb8f6154d..8454650ecf73f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -40,9 +40,9 @@ public class TestAutoCreatedQueueTemplate { @Before public void setUp() throws Exception { conf = new CapacitySchedulerConfiguration(); - conf.setQueues("root", new String[]{"a"}); - conf.setQueues("a", new String[]{"b"}); - conf.setQueues("b", new String[]{"c"}); + conf.setQueues(ROOT, new String[]{"a"}); + conf.setQueues(TEST_QUEUE_A, new String[]{"b"}); + conf.setQueues(TEST_QUEUE_B, new String[]{"c"}); } @@ -170,11 +170,11 @@ public void testQueueSpecificTemplates() { template.setTemplateEntriesForChild(conf, TEST_QUEUE_B, true); Assert.assertNull("default-node-label-expression is set for parent", - conf.getDefaultNodeLabelExpression(TEST_QUEUE_A.getFullPath())); + conf.getDefaultNodeLabelExpression(TEST_QUEUE_A)); Assert.assertEquals("default-node-label-expression is not set for leaf", - "test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B.getFullPath())); + "test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B)); Assert.assertFalse("auto queue removal is not disabled for parent", - conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A.getFullPath())); + conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A)); Assert.assertEquals("weight should not be overridden when set by " + "queue type specific template", 10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B), 10e-6); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java index 54a75200d4be6..5032e3be22eed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java @@ -71,6 +71,9 @@ * */ public class TestCSAllocateCustomResource { + private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A = new QueuePath(A_PATH); private YarnConfiguration conf; private RMNodeLabelsManager mgr; @@ -117,7 +120,7 @@ public void testCapacitySchedulerJobWhenConfigureCustomResourceType() .getConfigurationWithMultipleQueues(conf); newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class, ResourceCalculator.class); - newConf.set(QueuePrefixes.getQueuePrefix("root.a") + newConf.set(QueuePrefixes.getQueuePrefix(A) + MAXIMUM_ALLOCATION_MB, "4096"); // We must set this to false to avoid MockRM init configuration with // resource-types.xml by ResourceUtils.resetResourceTypes(conf); @@ -297,7 +300,7 @@ public void testCapacitySchedulerAbsoluteConfWithCustomResourceType() Resource.newInstance(1000, 10, valuesMax); // Define top-level queues - newConf.setQueues(CapacitySchedulerConfiguration.ROOT, + newConf.setQueues(ROOT, new String[] {"a", "b", "c"}); newConf.setMinimumResourceRequirement("", new QueuePath("root", "a"), aMINRES); @@ -315,10 +318,10 @@ public void testCapacitySchedulerAbsoluteConfWithCustomResourceType() CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); Assert.assertEquals(aMINRES, cs.getConfiguration(). - getMinimumResourceRequirement("", "root.a", resourceTypes)); + getMinimumResourceRequirement("", A, resourceTypes)); Assert.assertEquals(aMAXRES, cs.getConfiguration(). - getMaximumResourceRequirement("", "root.a", resourceTypes)); + getMaximumResourceRequirement("", A, resourceTypes)); // Check the gpu resource of queue is right. Assert.assertEquals(aMINRES, cs.getQueue("root.a"). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java index f7460de7aafd6..768768e95dbf9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java @@ -103,11 +103,15 @@ public void setup() throws IOException { } private void setupQueues(CapacitySchedulerConfiguration config) { - config.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {"queue1", "queue2"}); - config.setQueues("root.queue1", new String[] {"subqueue1", "subqueue2"}); - config.setQueues("root.queue1.subqueue1", new String[] {"leaf1"}); - config.setQueues("root.queue1.subqueue2", new String[] {"leaf2"}); + QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + QueuePath queue1 = new QueuePath("root.queue1"); + QueuePath subqueue1 = new QueuePath("root.queue1.subqueue1"); + QueuePath subqueue2 = new QueuePath("root.queue1.subqueue2"); + + config.setQueues(root, new String[] {"queue1", "queue2"}); + config.setQueues(queue1, new String[] {"subqueue1", "subqueue2"}); + config.setQueues(subqueue1, new String[] {"leaf1"}); + config.setQueues(subqueue2, new String[] {"leaf2"}); config.setFloat(PREFIX + "root.capacity", 100.0f); config.setFloat(PREFIX + "root.queue1.capacity", 50.0f); config.setFloat(PREFIX + "root.queue2.capacity", 50.0f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 762ca2af0e1ea..2eeb10ebe1fa2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -572,15 +572,14 @@ protected RMNodeLabelsManager createNodeLabelManager() { @Test public void testMaximumCapacitySetup() { float delta = 0.0000001f; - QueuePath queuePathA = new QueuePath(A); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE, - conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta); - conf.setMaximumCapacity(A_QUEUE_PATH, 50.0f); - assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta); - conf.setMaximumCapacity(A_QUEUE_PATH, -1); + conf.getNonLabeledQueueMaximumCapacity(A), delta); + conf.setMaximumCapacity(A, 50.0f); + assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(A), delta); + conf.setMaximumCapacity(A, -1); assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE, - conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta); + conf.getNonLabeledQueueMaximumCapacity(A), delta); } @Test @@ -591,8 +590,8 @@ public void testQueueMaximumAllocations() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); - setMaxAllocMb(conf, A1_QUEUE_PATH, 1024); - setMaxAllocVcores(conf, A1_QUEUE_PATH, 1); + setMaxAllocMb(conf, A1, 1024); + setMaxAllocVcores(conf, A1, 1); scheduler.init(conf); scheduler.start(); @@ -620,18 +619,19 @@ public void testQueueMaximumAllocations() { public void testParseQueueWithAbsoluteResource() { String childQueue = "testQueue"; String labelName = "testLabel"; + QueuePath childQueuePath = new QueuePath("root." + childQueue); CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.setQueues("root", new String[] {childQueue}); - conf.setCapacity(new QueuePath("root." + childQueue), "[memory=20480,vcores=200]"); - conf.setAccessibleNodeLabels("root." + childQueue, + conf.setQueues(ROOT, new String[] {childQueue}); + conf.setCapacity(childQueuePath, "[memory=20480,vcores=200]"); + conf.setAccessibleNodeLabels(childQueuePath, Sets.newHashSet(labelName)); - conf.setCapacityByLabel("root", labelName, "[memory=10240,vcores=100]"); - conf.setCapacityByLabel("root." + childQueue, labelName, + conf.setCapacityByLabel(ROOT, labelName, "[memory=10240,vcores=100]"); + conf.setCapacityByLabel(childQueuePath, labelName, "[memory=4096,vcores=10]"); cs.init(conf); @@ -1219,8 +1219,8 @@ null, new RMContainerTokenSecretManager(conf), cs.reinitialize(conf, rmContext); CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueB2 = findQueue(queueB, B2); + CSQueue queueB = findQueue(rootQueue, B_PATH); + CSQueue queueB2 = findQueue(queueB, B2_PATH); // When preemption turned on for the whole system // (yarn.resourcemanager.scheduler.monitor.enable=true), and with no other @@ -1231,7 +1231,7 @@ null, new RMContainerTokenSecretManager(conf), // Disable preemption at the root queue level. // The preemption property should be inherited from root all the // way down so that root.b.b2 should NOT be preemptable. - conf.setPreemptionDisabled(rootQueue.getQueuePath(), true); + conf.setPreemptionDisabled(rootQueue.getQueuePathObject(), true); cs.reinitialize(conf, rmContext); assertTrue( "queue " + B2 + " should have inherited non-preemptability from root", @@ -1239,8 +1239,8 @@ null, new RMContainerTokenSecretManager(conf), // Enable preemption for root (grandparent) but disable for root.b (parent). // root.b.b2 should inherit property from parent and NOT be preemptable - conf.setPreemptionDisabled(rootQueue.getQueuePath(), false); - conf.setPreemptionDisabled(queueB.getQueuePath(), true); + conf.setPreemptionDisabled(rootQueue.getQueuePathObject(), false); + conf.setPreemptionDisabled(queueB.getQueuePathObject(), true); cs.reinitialize(conf, rmContext); assertTrue( "queue " + B2 + " should have inherited non-preemptability from parent", @@ -1248,7 +1248,7 @@ null, new RMContainerTokenSecretManager(conf), // When preemption is turned on for root.b.b2, it should be preemptable // even though preemption is disabled on root.b (parent). - conf.setPreemptionDisabled(queueB2.getQueuePath(), false); + conf.setPreemptionDisabled(queueB2.getQueuePathObject(), false); cs.reinitialize(conf, rmContext); assertFalse("queue " + B2 + " should have been preemptable", queueB2.getPreemptionDisabled()); @@ -1528,18 +1528,18 @@ public void testParentQueueMaxCapsAreRespected() throws Exception { * A1 A2 */ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - csConf.setCapacity(A_QUEUE_PATH, 50); - csConf.setMaximumCapacity(A_QUEUE_PATH, 50); - csConf.setCapacity(B_QUEUE_PATH, 50); + csConf.setQueues(ROOT, new String[] {"a", "b"}); + csConf.setCapacity(A, 50); + csConf.setMaximumCapacity(A, 50); + csConf.setCapacity(B, 50); // Define 2nd-level queues csConf.setQueues(A, new String[] {"a1", "a2"}); - csConf.setCapacity(A1_QUEUE_PATH, 50); + csConf.setCapacity(A1, 50); csConf.setUserLimitFactor(A1, 100.0f); - csConf.setCapacity(A2_QUEUE_PATH, 50); + csConf.setCapacity(A2, 50); csConf.setUserLimitFactor(A2, 100.0f); - csConf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); + csConf.setCapacity(B1, B1_CAPACITY); csConf.setUserLimitFactor(B1, 100.0f); YarnConfiguration conf = new YarnConfiguration(csConf); @@ -2526,13 +2526,11 @@ private CapacityScheduler setUpCSQueue(long maxLifetime, long defaultLifetime) { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] {"default"}); - csConf.setCapacity(DEFAULT_QUEUE_PATH, 100); - csConf.setMaximumLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT + ".default", maxLifetime); - csConf.setDefaultLifetimePerQueue( - CapacitySchedulerConfiguration.ROOT + ".default", defaultLifetime); + csConf.setCapacity(DEFAULT, 100); + csConf.setMaximumLifetimePerQueue(DEFAULT, maxLifetime); + csConf.setDefaultLifetimePerQueue(DEFAULT, defaultLifetime); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler cs = new CapacityScheduler(); @@ -2865,11 +2863,11 @@ ImmutableMap. builder() .getMaxAbsoluteCapacity(), DELTA); // Add child queue to a, and reinitialize. Metrics should be updated - csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", + csConf.setQueues(A, new String[] {"a1", "a2", "a3"}); - csConf.setCapacity(A2_QUEUE_PATH, 29.5f); - csConf.setCapacity(A3_QUEUE_PATH, 40.5f); - csConf.setMaximumCapacity(A3_QUEUE_PATH, + csConf.setCapacity(A2, 29.5f); + csConf.setCapacity(A3, 40.5f); + csConf.setMaximumCapacity(A3, 50.0f); cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, @@ -2914,14 +2912,14 @@ null, new RMContainerTokenSecretManager(csConf), public void testReservedContainerLeakWhenMoveApplication() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] {"a", "b"}); - csConf.setCapacity(A_QUEUE_PATH, 50); - csConf.setMaximumCapacity(A_QUEUE_PATH, 100); - csConf.setUserLimitFactor("root.a", 100); - csConf.setCapacity(B_QUEUE_PATH, 50); - csConf.setMaximumCapacity(B_QUEUE_PATH, 100); - csConf.setUserLimitFactor("root.b", 100); + csConf.setCapacity(A, 50); + csConf.setMaximumCapacity(A, 100); + csConf.setUserLimitFactor(A, 100); + csConf.setCapacity(B, 50); + csConf.setMaximumCapacity(B, 100); + csConf.setUserLimitFactor(B, 100); YarnConfiguration conf=new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAmbiguousLeafs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAmbiguousLeafs.java index 69824e3c3fa44..1e29963fb92ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAmbiguousLeafs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAmbiguousLeafs.java @@ -29,14 +29,26 @@ import java.io.IOException; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; - public class TestCapacitySchedulerAmbiguousLeafs { /** * Internal counter for incremental application id generation */ int appId = 0; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath DEFAULT = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".default"); + private static final QueuePath A = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".a"); + private static final QueuePath B = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".b"); + private static final QueuePath A_UNIQUE = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".a.unique"); + private static final QueuePath A_AMBI = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".a.ambi"); + private static final QueuePath B_AMBI = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".b.ambi"); + /** * Helper method to submit applications via RMClientService, to make sure * all submissions go through RMAppManager. @@ -79,19 +91,19 @@ public void testAmbiguousSubmissionWithACL() throws Exception { schedulerConf.setAcl(ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); schedulerConf.setAcl(ROOT, QueueACL.ADMINISTER_QUEUE, "forbidden forbidden"); - schedulerConf.setQueues(ROOT + ".a", new String[] {"unique", "ambi"}); - schedulerConf.setAcl(ROOT + ".a", QueueACL.SUBMIT_APPLICATIONS, "forbidden forbidden"); - schedulerConf.setCapacity(ROOT + ".a", 45); + schedulerConf.setQueues(A, new String[] {"unique", "ambi"}); + schedulerConf.setAcl(A, QueueACL.SUBMIT_APPLICATIONS, "forbidden forbidden"); + schedulerConf.setCapacity(A, 45); - schedulerConf.setQueues(ROOT + ".b", new String[] {"ambi"}); - schedulerConf.setCapacity(ROOT + ".b", 45); - schedulerConf.setCapacity(ROOT + ".default", 10); + schedulerConf.setQueues(B, new String[] {"ambi"}); + schedulerConf.setCapacity(B, 45); + schedulerConf.setCapacity(DEFAULT, 10); - schedulerConf.setCapacity(ROOT + ".a.unique", 50); - schedulerConf.setAcl(ROOT + ".a.unique", QueueACL.SUBMIT_APPLICATIONS, "* *"); - schedulerConf.setCapacity(ROOT + ".a.ambi", 50); - schedulerConf.setAcl(ROOT + ".a.ambi", QueueACL.SUBMIT_APPLICATIONS, "* *"); - schedulerConf.setCapacity(ROOT + ".b.ambi", 100); + schedulerConf.setCapacity(A_UNIQUE, 50); + schedulerConf.setAcl(A_UNIQUE, QueueACL.SUBMIT_APPLICATIONS, "* *"); + schedulerConf.setCapacity(A_AMBI, 50); + schedulerConf.setAcl(A_AMBI, QueueACL.SUBMIT_APPLICATIONS, "* *"); + schedulerConf.setCapacity(B_AMBI, 100); schedulerConf.set(CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT, "json"); //Simple %specified mapping rule for all submissions with skip fallback diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java index ac4be49359efd..e881c3327b672 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java @@ -97,6 +97,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper; @@ -1173,7 +1174,7 @@ public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception { new CapacitySchedulerConfiguration(conf); // Define top-level queues - newConf.setQueues(CapacitySchedulerConfiguration.ROOT, + newConf.setQueues(ROOT, new String[]{"a", "b"}); newConf.setCapacity(A, 50); @@ -1469,7 +1470,7 @@ public void testAppSubmission() throws Exception { ResourceScheduler.class); conf.setQueues(A, new String[]{"a1", "a2", "b"}); conf.setCapacity(A1, 20); - conf.setCapacity("root.a.b", 10); + conf.setCapacity(new QueuePath("root.a.b"), 10); MockRM rm = new MockRM(conf); rm.start(); @@ -1544,7 +1545,7 @@ private Configuration getCapacityConfiguration(Configuration config) { new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[]{"a", "b"}); conf.setCapacity(A, 50); conf.setCapacity(B, 50); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index 61fcd68f703ad..e9f162a8e3ffc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -96,8 +96,6 @@ .capacity.CapacitySchedulerConfiguration.DOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler .capacity.CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler - .capacity.CapacitySchedulerConfiguration.ROOT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -111,40 +109,43 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { public static final ContainerUpdates NULL_UPDATE_REQUESTS = new ContainerUpdates(); - public static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - public static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; - public static final String C = CapacitySchedulerConfiguration.ROOT + ".c"; - public static final String D = CapacitySchedulerConfiguration.ROOT + ".d"; - public static final String E = CapacitySchedulerConfiguration.ROOT + ".e"; - - public static final QueuePath A_QUEUE_PATH = new QueuePath(A); - public static final QueuePath B_QUEUE_PATH = new QueuePath(B); - public static final QueuePath C_QUEUE_PATH = new QueuePath(C); - public static final QueuePath D_QUEUE_PATH = new QueuePath(D); - public static final QueuePath E_QUEUE_PATH = new QueuePath(E); - public static final String ESUBGROUP1 = + public static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default"; + public static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + public static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + public static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c"; + public static final String D_PATH = CapacitySchedulerConfiguration.ROOT + ".d"; + public static final String E_PATH = CapacitySchedulerConfiguration.ROOT + ".e"; + + public static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + public static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH); + public static final QueuePath A = new QueuePath(A_PATH); + public static final QueuePath B = new QueuePath(B_PATH); + public static final QueuePath C = new QueuePath(C_PATH); + public static final QueuePath D = new QueuePath(D_PATH); + public static final QueuePath E = new QueuePath(E_PATH); + public static final String ESUBGROUP1_PATH = CapacitySchedulerConfiguration.ROOT + ".esubgroup1"; - public static final String FGROUP = + public static final String FGROUP_PATH = CapacitySchedulerConfiguration.ROOT + ".fgroup"; - public static final String A1 = A + ".a1"; - public static final String A2 = A + ".a2"; - public static final String B1 = B + ".b1"; - public static final String B2 = B + ".b2"; - public static final String B3 = B + ".b3"; - public static final String B4 = B + ".b4subgroup1"; - public static final String ESUBGROUP1_A = ESUBGROUP1 + ".e"; - public static final String FGROUP_F = FGROUP + ".f"; - - public static final QueuePath A1_QUEUE_PATH = new QueuePath(A1); - public static final QueuePath A2_QUEUE_PATH = new QueuePath(A2); - public static final QueuePath B1_QUEUE_PATH = new QueuePath(B1); - public static final QueuePath B2_QUEUE_PATH = new QueuePath(B2); - public static final QueuePath B3_QUEUE_PATH = new QueuePath(B3); - public static final QueuePath B4_QUEUE_PATH = new QueuePath(B4); - public static final QueuePath E_GROUP_QUEUE_PATH = new QueuePath(ESUBGROUP1); - public static final QueuePath F_GROUP_QUEUE_PATH = new QueuePath(FGROUP); - public static final QueuePath E_SG_QUEUE_PATH = new QueuePath(ESUBGROUP1_A); - public static final QueuePath F_SG_QUEUE_PATH = new QueuePath(FGROUP_F); + public static final String A1_PATH = A_PATH + ".a1"; + public static final String A2_PATH = A_PATH + ".a2"; + public static final String B1_PATH = B_PATH + ".b1"; + public static final String B2_PATH = B_PATH + ".b2"; + public static final String B3_PATH = B_PATH + ".b3"; + public static final String B4_PATH = B_PATH + ".b4subgroup1"; + public static final String ESUBGROUP1_A_PATH = ESUBGROUP1_PATH + ".e"; + public static final String FGROUP_F_PATH = FGROUP_PATH + ".f"; + + public static final QueuePath A1 = new QueuePath(A1_PATH); + public static final QueuePath A2 = new QueuePath(A2_PATH); + public static final QueuePath B1 = new QueuePath(B1_PATH); + public static final QueuePath B2 = new QueuePath(B2_PATH); + public static final QueuePath B3 = new QueuePath(B3_PATH); + public static final QueuePath B4 = new QueuePath(B4_PATH); + public static final QueuePath E_GROUP = new QueuePath(ESUBGROUP1_PATH); + public static final QueuePath F_GROUP = new QueuePath(FGROUP_PATH); + public static final QueuePath E_SG = new QueuePath(ESUBGROUP1_A_PATH); + public static final QueuePath F_SG = new QueuePath(FGROUP_F_PATH); public static final float A_CAPACITY = 20f; public static final float B_CAPACITY = 20f; @@ -389,43 +390,43 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( new String[] {"a", "b", "c", "d", "esubgroup1", "esubgroup2", "fgroup", "a1group", "ggroup", "g"}); - conf.setCapacity(A_QUEUE_PATH, A_CAPACITY); - conf.setCapacity(B_QUEUE_PATH, B_CAPACITY); - conf.setCapacity(C_QUEUE_PATH, C_CAPACITY); - conf.setCapacity(D_QUEUE_PATH, D_CAPACITY); - conf.setCapacity(E_GROUP_QUEUE_PATH, ESUBGROUP1_CAPACITY); - conf.setCapacity(F_GROUP_QUEUE_PATH, FGROUP_CAPACITY); + conf.setCapacity(A, A_CAPACITY); + conf.setCapacity(B, B_CAPACITY); + conf.setCapacity(C, C_CAPACITY); + conf.setCapacity(D, D_CAPACITY); + conf.setCapacity(E_GROUP, ESUBGROUP1_CAPACITY); + conf.setCapacity(F_GROUP, FGROUP_CAPACITY); // Define 2nd-level queues conf.setQueues(A, new String[] { "a1", "a2" }); - conf.setCapacity(A1_QUEUE_PATH, A1_CAPACITY); + conf.setCapacity(A1, A1_CAPACITY); conf.setUserLimitFactor(A1, 100.0f); - conf.setCapacity(A2_QUEUE_PATH, A2_CAPACITY); + conf.setCapacity(A2, A2_CAPACITY); conf.setUserLimitFactor(A2, 100.0f); conf.setQueues(B, new String[] { "b1", "b2", "b3", "b4subgroup1" }); - conf.setCapacity(B1_QUEUE_PATH, B1_CAPACITY); + conf.setCapacity(B1, B1_CAPACITY); conf.setUserLimitFactor(B1, 100.0f); - conf.setCapacity(B2_QUEUE_PATH, B2_CAPACITY); + conf.setCapacity(B2, B2_CAPACITY); conf.setUserLimitFactor(B2, 100.0f); - conf.setCapacity(B3_QUEUE_PATH, B3_CAPACITY); + conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); - conf.setCapacity(B4_QUEUE_PATH, B4_CAPACITY); + conf.setCapacity(B4, B4_CAPACITY); conf.setUserLimitFactor(B4, 100.0f); - conf.setQueues(ESUBGROUP1, new String[] {"e"}); - conf.setCapacity(E_SG_QUEUE_PATH, 100f); - conf.setUserLimitFactor(ESUBGROUP1_A, 100.0f); - conf.setQueues(FGROUP, new String[] {"f"}); - conf.setCapacity(F_SG_QUEUE_PATH, 100f); - conf.setUserLimitFactor(FGROUP_F, 100.0f); + conf.setQueues(E_GROUP, new String[] {"e"}); + conf.setCapacity(E_SG, 100f); + conf.setUserLimitFactor(E_SG, 100.0f); + conf.setQueues(F_GROUP, new String[] {"f"}); + conf.setCapacity(F_SG, 100f); + conf.setUserLimitFactor(F_SG, 100.0f); conf.setUserLimitFactor(C, 1.0f); conf.setAutoCreateChildQueueEnabled(C, true); //Setup leaf queue template configs conf.setAutoCreatedLeafQueueConfigCapacity(C, 50.0f); - conf.setAutoCreatedLeafQueueConfigMaxCapacity(C_QUEUE_PATH, 100.0f); + conf.setAutoCreatedLeafQueueConfigMaxCapacity(C, 100.0f); conf.setAutoCreatedLeafQueueConfigUserLimit(C, 100); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f); @@ -454,7 +455,7 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( //Setup leaf queue template configs conf.setAutoCreatedLeafQueueConfigCapacity(D, 10.0f); - conf.setAutoCreatedLeafQueueConfigMaxCapacity(D_QUEUE_PATH, 100.0f); + conf.setAutoCreatedLeafQueueConfigMaxCapacity(D, 100.0f); conf.setAutoCreatedLeafQueueConfigUserLimit(D, 3); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(D, 100); @@ -489,16 +490,16 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration( //setup new queues with one of them auto enabled // Define top-level queues // Set childQueue for root - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] {"c"}); - conf.setCapacity(C_QUEUE_PATH, 100f); + conf.setCapacity(C, 100f); conf.setUserLimitFactor(C, 1.0f); conf.setAutoCreateChildQueueEnabled(C, true); //Setup leaf queue template configs conf.setAutoCreatedLeafQueueConfigCapacity(C, 100f); - conf.setAutoCreatedLeafQueueConfigMaxCapacity(C_QUEUE_PATH, 100.0f); + conf.setAutoCreatedLeafQueueConfigMaxCapacity(C, 100.0f); conf.setAutoCreatedLeafQueueConfigUserLimit(C, 100); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f); @@ -800,7 +801,7 @@ protected void validateEffectiveMinResource(ResourceManager rm, if (expectedQueueEntitlements.get(label).getCapacity() > EPSILON) { if (leafQueue.getCapacityConfigType().equals(ABSOLUTE_RESOURCE)) { - String templatePrefix = QueuePrefixes.getAutoCreatedQueueTemplateConfPrefix( + QueuePath templatePrefix = QueuePrefixes.getAutoCreatedQueueObjectTemplateConfPrefix( parentQueue.getQueuePathObject()); Resource resourceTemplate = parentQueue.getLeafQueueTemplate().getLeafQueueConfigs() .getMinimumResourceRequirement(label, templatePrefix, RESOURCE_TYPES); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java index a025f1ed7de0f..4a3b5d9564d3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java @@ -26,6 +26,8 @@ .capacity.TestCapacitySchedulerAutoCreatedQueueBase.D; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler .capacity.TestCapacitySchedulerAutoCreatedQueueBase.E; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler + .capacity.TestCapacitySchedulerAutoCreatedQueueBase.ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler .capacity.TestCapacitySchedulerAutoCreatedQueueBase.USER0; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler @@ -60,7 +62,7 @@ public void setUp() throws Exception { //setup new queues with one of them auto enabled // Define top-level queues // Set childQueue for root - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "c" }); + conf.setQueues(ROOT, new String[] { "c" }); conf.setCapacity(C, 100f); conf.setUserLimitFactor(C, 1.0f); @@ -91,7 +93,7 @@ public void setUp() throws Exception { //setup new queues with one of them auto enabled // Define top-level queues // Set childQueue for root - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] { "c", "d", "e" }); conf.setCapacity(C, 45f); conf.setCapacity(D, 45f); @@ -120,8 +122,8 @@ public void setUp() throws Exception { conf.setAutoCreatedLeafQueueConfigUserLimit(E, 100); conf.setAutoCreatedLeafQueueConfigUserLimitFactor(E, 3.0f); - conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1); - conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".d", 2); + conf.setQueuePriority(C, 1); + conf.setQueuePriority(D, 2); return conf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index 90c63dbd2b95f..68e4acfc244e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -578,7 +578,7 @@ public void testAutoQueueCreationFailsForEmptyPathWithAQCAndWeightMode() //And weight mode, to allow dynamic auto queue creation for root CapacitySchedulerConfiguration conf = setupSchedulerConfiguration(); conf.setAutoQueueCreationV2Enabled(ROOT, true); - conf.setCapacity("root.default", "1w"); + conf.setCapacity(DEFAULT, "1w"); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); @@ -996,15 +996,14 @@ public void testDynamicAutoQueueCreationWithTags() try { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {"a", "b"}); - csConf.setCapacity("root.a", 90); - csConf.setCapacity("root.b", 10); - csConf.setAutoCreateChildQueueEnabled("root.a", true); - csConf.setAutoCreatedLeafQueueConfigCapacity("root.a", 50); - csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.a", 100); - csConf.setAcl("root.a", QueueACL.ADMINISTER_QUEUE, "*"); - csConf.setAcl("root.a", QueueACL.SUBMIT_APPLICATIONS, "*"); + csConf.setQueues(ROOT, new String[] {"a", "b"}); + csConf.setCapacity(A, 90); + csConf.setCapacity(B, 10); + csConf.setAutoCreateChildQueueEnabled(A, true); + csConf.setAutoCreatedLeafQueueConfigCapacity(A, 50); + csConf.setAutoCreatedLeafQueueConfigMaxCapacity(A, 100); + csConf.setAcl(A, QueueACL.ADMINISTER_QUEUE, "*"); + csConf.setAcl(A, QueueACL.SUBMIT_APPLICATIONS, "*"); csConf.setBoolean(YarnConfiguration .APPLICATION_TAG_BASED_PLACEMENT_ENABLED, true); csConf.setStrings(YarnConfiguration diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java index 1bee66eb9db9a..291591b6213c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java @@ -598,10 +598,10 @@ private CapacitySchedulerConfiguration setupCSConfiguration(YarnConfiguration co csConf.set(YarnConfiguration.RESOURCE_TYPES, ResourceInformation.GPU_URI); } - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(new QueuePath(CapacitySchedulerConfiguration.ROOT), new String[]{PARENT_A, PARENT_B}); - csConf.setQueues(PARENT_A_FULL_PATH.getFullPath(), new String[]{LEAF_A}); - csConf.setQueues(PARENT_B_FULL_PATH.getFullPath(), new String[]{LEAF_B}); + csConf.setQueues(PARENT_A_FULL_PATH, new String[]{LEAF_A}); + csConf.setQueues(PARENT_B_FULL_PATH, new String[]{LEAF_B}); if (useDominantRC) { setupGpuResourceValues(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java index b1534e3f96593..85e0ff494eb93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java @@ -24,14 +24,15 @@ import java.util.Set; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class TestCapacitySchedulerConfiguration { - private static final String ROOT_TEST = ROOT + ".test"; + private static final String ROOT_TEST_PATH = CapacitySchedulerConfiguration.ROOT + ".test"; + private static final QueuePath ROOT_TEST = new QueuePath(ROOT_TEST_PATH); + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); private static final String EMPTY_ACL = ""; private static final String SPACE_ACL = " "; private static final String USER1 = "user1"; @@ -46,29 +47,29 @@ private CapacitySchedulerConfiguration createDefaultCsConf() { return new CapacitySchedulerConfiguration(new Configuration(false), false); } - private AccessControlList getSubmitAcl(CapacitySchedulerConfiguration csConf, String queue) { + private AccessControlList getSubmitAcl(CapacitySchedulerConfiguration csConf, QueuePath queue) { return csConf.getAcl(queue, QueueACL.SUBMIT_APPLICATIONS); } - private void setSubmitAppsConfig(CapacitySchedulerConfiguration csConf, String queue, + private void setSubmitAppsConfig(CapacitySchedulerConfiguration csConf, QueuePath queue, String value) { csConf.set(getSubmitAppsConfigKey(queue), value); } - private String getSubmitAppsConfigKey(String queue) { + private String getSubmitAppsConfigKey(QueuePath queue) { return QueuePrefixes.getQueuePrefix(queue) + "acl_submit_applications"; } - private void testWithGivenAclNoOneHasAccess(String queue, String aclValue) { + private void testWithGivenAclNoOneHasAccess(QueuePath queue, String aclValue) { testWithGivenAclNoOneHasAccessInternal(queue, queue, aclValue); } - private void testWithGivenAclNoOneHasAccess(String queueToSet, String queueToVerify, + private void testWithGivenAclNoOneHasAccess(QueuePath queueToSet, QueuePath queueToVerify, String aclValue) { testWithGivenAclNoOneHasAccessInternal(queueToSet, queueToVerify, aclValue); } - private void testWithGivenAclNoOneHasAccessInternal(String queueToSet, String queueToVerify, + private void testWithGivenAclNoOneHasAccessInternal(QueuePath queueToSet, QueuePath queueToVerify, String aclValue) { CapacitySchedulerConfiguration csConf = createDefaultCsConf(); setSubmitAppsConfig(csConf, queueToSet, aclValue); @@ -78,14 +79,14 @@ private void testWithGivenAclNoOneHasAccessInternal(String queueToSet, String qu assertFalse(acl.isAllAllowed()); } - private void testWithGivenAclCorrectUserAndGroupHasAccess(String queue, String aclValue, + private void testWithGivenAclCorrectUserAndGroupHasAccess(QueuePath queue, String aclValue, Set expectedUsers, Set expectedGroups) { testWithGivenAclCorrectUserAndGroupHasAccessInternal(queue, queue, aclValue, expectedUsers, expectedGroups); } - private void testWithGivenAclCorrectUserAndGroupHasAccessInternal(String queueToSet, - String queueToVerify, String aclValue, Set expectedUsers, + private void testWithGivenAclCorrectUserAndGroupHasAccessInternal(QueuePath queueToSet, + QueuePath queueToVerify, String aclValue, Set expectedUsers, Set expectedGroups) { CapacitySchedulerConfiguration csConf = createDefaultCsConf(); setSubmitAppsConfig(csConf, queueToSet, aclValue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index 7cfd457cb76b1..9260e81bf7083 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -18,20 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.getDefaultCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -287,7 +274,7 @@ public void testMoveAppToPlanQueue() throws Exception { private void setupPlanQueueConfiguration(CapacitySchedulerConfiguration conf) { - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] { "a", "b" }); conf.setCapacity(A, A_CAPACITY); @@ -302,7 +289,7 @@ private void setupPlanQueueConfiguration(CapacitySchedulerConfiguration conf) { conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); - conf.setReservable(A, true); + conf.setReservable(A.getFullPath(), true); conf.setReservationWindow(A, 86400 * 1000); conf.setAverageCapacity(A, 1.0f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java index ee6a882a37709..3252cf4fe2159 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java @@ -650,7 +650,8 @@ public void testPreemptionConsidersUserLimit() * from app1, and app2 will receive the preempted container */ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); - csConf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".c", 0.1f); + QueuePath cQueuePath = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".c"); + csConf.setUserLimitFactor(cQueuePath, 0.1f); MockRM rm1 = new MockRM(csConf); rm1.getRMContext().setNodeLabelManager(mgr); rm1.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java index b2b304d4f8e28..c445b90f09cbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.waitforNMRegistered; @@ -61,6 +60,7 @@ public class TestCapacitySchedulerMultiNodes { private static final Logger LOG = LoggerFactory .getLogger(TestCapacitySchedulerMultiNodes.class); + private static final QueuePath DEFAULT = new QueuePath("root.default"); private CapacitySchedulerConfiguration conf; private static final String POLICY_CLASS_NAME = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.ResourceUsageMultiNodeLookupPolicy"; @@ -201,7 +201,7 @@ public void testExcessReservationWillBeUnreserved() throws Exception { YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + ".resource-based.sorting-interval.ms", 0); - newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, + newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT, 1.0f); MockRM rm1 = new MockRM(newConf); @@ -287,7 +287,7 @@ public void testAllocateForReservedContainer() throws Exception { YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + ".resource-based.sorting-interval.ms", 0); - newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, + newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT, 1.0f); MockRM rm1 = new MockRM(newConf); @@ -358,7 +358,7 @@ public void testAllocateOfReservedContainerFromAnotherNode() YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + ".resource-based.sorting-interval.ms", 0); - newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, + newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT, 1.0f); MockRM rm1 = new MockRM(newConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index b77ae2f49c09a..29b0aa714786f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -50,13 +50,17 @@ import java.io.IOException; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; - public class TestCapacitySchedulerNewQueueAutoCreation extends TestCapacitySchedulerAutoCreatedQueueBase { private static final Logger LOG = LoggerFactory.getLogger( org.apache.hadoop.yarn.server.resourcemanager .scheduler.capacity.TestCapacitySchedulerAutoCreatedQueueBase.class); + private static final QueuePath EMPTY_AUTO_PARENT = new QueuePath("root.empty-auto-parent"); + private static final QueuePath A_A2_AUTO = new QueuePath("root.a.a2-auto"); + private static final QueuePath E_AUTO = new QueuePath("root.e-auto"); + private static final QueuePath E_E1 = new QueuePath("root.e.e1"); + private static final QueuePath A_A_AUTO_A2 = new QueuePath("root.a.a-auto.a2"); + private static final QueuePath A_A1_AUTO_A2_AUTO = new QueuePath("root.a.a1-auto.a2-auto"); public static final int GB = 1024; private static final int MAX_MEMORY = 1200; private MockRM mockRM = null; @@ -65,7 +69,6 @@ public class TestCapacitySchedulerNewQueueAutoCreation private CapacitySchedulerQueueManager autoQueueHandler; private AutoCreatedQueueDeletionPolicy policy = new AutoCreatedQueueDeletionPolicy(); - private QueuePath queuePath public CapacityScheduler getCs() { return cs; @@ -90,16 +93,16 @@ public void setUp() throws Exception { ResourceScheduler.class); // By default, set 3 queues, a/b, and a.a1 - csConf.setQueues("root", new String[]{"a", "b"}); - csConf.setNonLabeledQueueWeight(ROOT_QUEUE_PATH, 1f); - csConf.setNonLabeledQueueWeight(A_QUEUE_PATH, 1f); - csConf.setNonLabeledQueueWeight(B_QUEUE_PATH, 1f); - csConf.setQueues("root.a", new String[]{"a1"}); - csConf.setNonLabeledQueueWeight(A1_QUEUE_PATH, 1f); - csConf.setAutoQueueCreationV2Enabled("root", true); - csConf.setAutoQueueCreationV2Enabled("root.a", true); - csConf.setAutoQueueCreationV2Enabled("root.e", true); - csConf.setAutoQueueCreationV2Enabled(PARENT_QUEUE, true); + csConf.setQueues(ROOT, new String[]{"a", "b"}); + csConf.setNonLabeledQueueWeight(ROOT, 1f); + csConf.setNonLabeledQueueWeight(A, 1f); + csConf.setNonLabeledQueueWeight(B, 1f); + csConf.setQueues(A, new String[]{"a1"}); + csConf.setNonLabeledQueueWeight(A1, 1f); + csConf.setAutoQueueCreationV2Enabled(ROOT, true); + csConf.setAutoQueueCreationV2Enabled(A, true); + csConf.setAutoQueueCreationV2Enabled(E, true); + csConf.setAutoQueueCreationV2Enabled(new QueuePath(PARENT_QUEUE), true); // Test for auto deletion when expired csConf.setAutoExpiredDeletionTime(1); } @@ -280,7 +283,7 @@ public void testAutoCreateQueueStaticParentOneLevel() throws Exception { public void testAutoCreateQueueAutoParentTwoLevelsWithSiblings() throws Exception { startScheduler(); - csConf.setAutoQueueCreationV2Enabled("root.a.a2-auto", true); + csConf.setAutoQueueCreationV2Enabled(A_A2_AUTO, true); // root.a has 1/2 abs resource -> a1 and a2-auto same weight 1/4 // -> a3-auto is alone with weight 1/4 @@ -314,9 +317,9 @@ public void testAutoCreateQueueShouldFailWhenNonParentQueue() public void testAutoCreateQueueWhenSiblingsNotInWeightMode() throws Exception { startScheduler(); - csConf.setCapacity("root.a", 50f); - csConf.setCapacity("root.b", 50f); - csConf.setCapacity("root.a.a1", 100f); + csConf.setCapacity(A, 50f); + csConf.setCapacity(B, 50f); + csConf.setCapacity(A1, 100f); cs.reinitialize(csConf, mockRM.getRMContext()); createQueue("root.a.a2-auto"); } @@ -330,7 +333,7 @@ public void testAutoCreateMaximumQueueDepth() () -> createQueue("root.a.a3-auto.a4-auto.a5-auto")); // Set depth 3 for root.a, making it a valid scenario - csConf.setMaximumAutoCreatedQueueDepth(A_QUEUE_PATH, 3); + csConf.setMaximumAutoCreatedQueueDepth(A, 3); cs.reinitialize(csConf, mockRM.getRMContext()); try { createQueue("root.a.a3-auto.a4-auto.a5-auto"); @@ -341,7 +344,7 @@ public void testAutoCreateMaximumQueueDepth() // Set global depth to 3 csConf.setMaximumAutoCreatedQueueDepth(3); - csConf.unset(QueuePrefixes.getQueuePrefix(A_QUEUE_PATH) + csConf.unset(QueuePrefixes.getQueuePrefix(A) + CapacitySchedulerConfiguration.MAXIMUM_QUEUE_DEPTH); cs.reinitialize(csConf, mockRM.getRMContext()); try { @@ -361,7 +364,7 @@ public void testAutoCreateMaximumQueueDepth() public void testAutoCreateQueueShouldFailIfNotEnabledForParent() throws Exception { startScheduler(); - csConf.setAutoQueueCreationV2Enabled("root", false); + csConf.setAutoQueueCreationV2Enabled(ROOT, false); cs.reinitialize(csConf, mockRM.getRMContext()); createQueue("root.c-auto"); } @@ -407,7 +410,7 @@ public void testConvertDynamicToStaticQueue() throws Exception { Assert.assertEquals(6f, a.getQueueCapacities().getWeight(), 1e-6); // Set queue c-auto's weight to 6, and mark c-auto to be static queue - csConf.setQueues("root", new String[]{"a", "b", "c-auto"}); + csConf.setQueues(ROOT, new String[]{"a", "b", "c-auto"}); csConf.setNonLabeledQueueWeight(new QueuePath("root.c-auto"), 6f); cs.reinitialize(csConf, mockRM.getRMContext()); @@ -425,9 +428,9 @@ public void testConvertDynamicToStaticQueue() throws Exception { createQueue("root.e-auto.e2-auto"); // Do change 2nd level queue from dynamic to static - csConf.setQueues("root", new String[]{"a", "b", "c-auto", "e-auto"}); + csConf.setQueues(ROOT, new String[]{"a", "b", "c-auto", "e-auto"}); csConf.setNonLabeledQueueWeight(new QueuePath("root.e-auto"), 6f); - csConf.setQueues("root.e-auto", new String[]{"e1-auto"}); + csConf.setQueues(E_AUTO, new String[]{"e1-auto"}); csConf.setNonLabeledQueueWeight(new QueuePath("root.e-auto.e1-auto"), 6f); cs.reinitialize(csConf, mockRM.getRMContext()); @@ -455,7 +458,7 @@ public void testConvertDynamicToStaticQueue() throws Exception { public void testConvertDynamicParentToStaticParent() throws Exception { startScheduler(); createQueue("root.d-auto.d1-auto"); - csConf.setQueues("root", new String[]{"a", "b", "d-auto"}); + csConf.setQueues(ROOT, new String[]{"a", "b", "d-auto"}); csConf.setNonLabeledQueueWeight(new QueuePath("root.a"), 6f); csConf.setNonLabeledQueueWeight(new QueuePath("root.d-auto"), 1f); cs.reinitialize(csConf, mockRM.getRMContext()); @@ -492,14 +495,14 @@ public void testAutoQueueCreationOnAppSubmission() throws Exception { public void testChildlessParentQueueWhenAutoQueueCreationEnabled() throws Exception { startScheduler(); - csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"}); - csConf.setNonLabeledQueueWeight(ROOT_QUEUE_PATH, 1f); - csConf.setNonLabeledQueueWeight(A_QUEUE_PATH, 1f); - csConf.setNonLabeledQueueWeight(B_QUEUE_PATH, 1f); - csConf.setQueues("root.a", new String[]{"a1"}); - csConf.setNonLabeledQueueWeight(A1_QUEUE_PATH, 1f); - csConf.setAutoQueueCreationV2Enabled("root", true); - csConf.setAutoQueueCreationV2Enabled("root.a", true); + csConf.setQueues(ROOT, new String[]{"a", "b", "empty-auto-parent"}); + csConf.setNonLabeledQueueWeight(ROOT, 1f); + csConf.setNonLabeledQueueWeight(A, 1f); + csConf.setNonLabeledQueueWeight(B, 1f); + csConf.setQueues(A, new String[]{"a1"}); + csConf.setNonLabeledQueueWeight(A1, 1f); + csConf.setAutoQueueCreationV2Enabled(ROOT, true); + csConf.setAutoQueueCreationV2Enabled(A, true); cs.reinitialize(csConf, mockRM.getRMContext()); CSQueue empty = cs.getQueue("root.empty-auto-parent"); @@ -507,15 +510,15 @@ public void testChildlessParentQueueWhenAutoQueueCreationEnabled() empty instanceof LeafQueue); empty.stopQueue(); - csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"}); - csConf.setNonLabeledQueueWeight(ROOT_QUEUE_PATH, 1f); - csConf.setNonLabeledQueueWeight(A_QUEUE_PATH, 1f); - csConf.setNonLabeledQueueWeight(B_QUEUE_PATH, 1f); - csConf.setQueues("root.a", new String[]{"a1"}); - csConf.setNonLabeledQueueWeight(A1_QUEUE_PATH, 1f); - csConf.setAutoQueueCreationV2Enabled("root", true); - csConf.setAutoQueueCreationV2Enabled("root.a", true); - csConf.setAutoQueueCreationV2Enabled("root.empty-auto-parent", true); + csConf.setQueues(ROOT, new String[]{"a", "b", "empty-auto-parent"}); + csConf.setNonLabeledQueueWeight(ROOT, 1f); + csConf.setNonLabeledQueueWeight(A, 1f); + csConf.setNonLabeledQueueWeight(B, 1f); + csConf.setQueues(A, new String[]{"a1"}); + csConf.setNonLabeledQueueWeight(A1, 1f); + csConf.setAutoQueueCreationV2Enabled(ROOT, true); + csConf.setAutoQueueCreationV2Enabled(A, true); + csConf.setAutoQueueCreationV2Enabled(EMPTY_AUTO_PARENT, true); cs.reinitialize(csConf, mockRM.getRMContext()); empty = cs.getQueue("root.empty-auto-parent"); @@ -693,7 +696,7 @@ public void testAutoCreateQueueIfAmbiguousQueueNames() throws Exception { public void testAutoCreateQueueMaxQueuesLimit() throws Exception { startScheduler(); - csConf.setAutoCreatedQueuesV2MaxChildQueuesLimit("root.e", 5); + csConf.setAutoCreatedQueuesV2MaxChildQueuesLimit(E, 5); cs.reinitialize(csConf, mockRM.getRMContext()); for (int i = 0; i < 5; ++i) { @@ -763,8 +766,8 @@ public void testAutoCreatedQueueTemplateConfig() throws Exception { // Set the capacity of label TEST csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix( cQueuePath) + "accessible-node-labels.TEST.capacity", "6w"); - csConf.setQueues("root", new String[]{"a", "b", "c"}); - csConf.setAutoQueueCreationV2Enabled("root.c", true); + csConf.setQueues(ROOT, new String[]{"a", "b", "c"}); + csConf.setAutoQueueCreationV2Enabled(C, true); cs.reinitialize(csConf, mockRM.getRMContext()); AbstractLeafQueue c1 = createQueue("root.c.c1"); Assert.assertEquals("weight is not set for label TEST", 6f, @@ -786,12 +789,12 @@ public void testAutoCreatedQueueConfigChange() throws Exception { a2.getQueueCapacities().getWeight(), 1e-6); a2 = (AbstractLeafQueue) cs.getQueue("root.a.a-auto.a2"); - csConf.setState("root.a.a-auto.a2", QueueState.STOPPED); + csConf.setState(A_A_AUTO_A2, QueueState.STOPPED); cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals("root.a.a-auto.a2 has not been stopped", QueueState.STOPPED, a2.getState()); - csConf.setState("root.a.a-auto.a2", QueueState.RUNNING); + csConf.setState(A_A_AUTO_A2, QueueState.RUNNING); cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals("root.a.a-auto.a2 is not running", QueueState.RUNNING, a2.getState()); @@ -802,9 +805,9 @@ public void testAutoCreateQueueState() throws Exception { startScheduler(); createQueue("root.e.e1"); - csConf.setState("root.e", QueueState.STOPPED); - csConf.setState("root.e.e1", QueueState.STOPPED); - csConf.setState("root.a", QueueState.STOPPED); + csConf.setState(E, QueueState.STOPPED); + csConf.setState(E_E1, QueueState.STOPPED); + csConf.setState(A, QueueState.STOPPED); cs.reinitialize(csConf, mockRM.getRMContext()); // Make sure the static queue is stopped @@ -820,13 +823,13 @@ public void testAutoCreateQueueState() throws Exception { QueueState.STOPPED); // Make root.e state to RUNNING - csConf.setState("root.e", QueueState.RUNNING); + csConf.setState(E, QueueState.RUNNING); cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals(cs.getQueue("root.e.e1").getState(), QueueState.STOPPED); // Make root.e.e1 state to RUNNING - csConf.setState("root.e.e1", QueueState.RUNNING); + csConf.setState(E_E1, QueueState.RUNNING); cs.reinitialize(csConf, mockRM.getRMContext()); Assert.assertEquals(cs.getQueue("root.e.e1").getState(), QueueState.RUNNING); @@ -952,7 +955,7 @@ public void testCapacitySchedulerAutoQueueDeletionDisabled() startScheduler(); // Test for disabled auto deletion csConf.setAutoExpiredDeletionEnabled( - "root.a.a1-auto.a2-auto", false); + A_A1_AUTO_A2_AUTO, false); csConf.setBoolean( YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); csConf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, @@ -1019,7 +1022,7 @@ public void testCapacitySchedulerAutoQueueDeletionDisabled() // Enabled now // The auto deletion will work. csConf.setAutoExpiredDeletionEnabled( - "root.a.a1-auto.a2-auto", true); + A_A1_AUTO_A2_AUTO, true); cs.reinitialize(csConf, mockRM.getRMContext()); // Wait the a2 deleted. @@ -1164,12 +1167,12 @@ public void testRemoveDanglingAutoCreatedQueuesOnReinit() throws Exception { getQueue("root.a.a-auto"); Assert.assertTrue(aAuto.isDynamicQueue()); - csConf.setState("root.a", QueueState.STOPPED); + csConf.setState(A, QueueState.STOPPED); cs.reinitialize(csConf, mockRM.getRMContext()); aAuto = (AbstractCSQueue) cs. getQueue("root.a.a-auto"); Assert.assertEquals("root.a.a-auto is not in STOPPED state", QueueState.STOPPED, aAuto.getState()); - csConf.setQueues("root", new String[]{"b"}); + csConf.setQueues(ROOT, new String[]{"b"}); cs.reinitialize(csConf, mockRM.getRMContext()); CSQueue aAutoNew = cs.getQueue("root.a.a-auto"); Assert.assertNull(aAutoNew); @@ -1179,24 +1182,24 @@ public void testRemoveDanglingAutoCreatedQueuesOnReinit() throws Exception { Assert.assertNotNull(aAutoNew); // Validate static grandparent deletion - csConf.setQueues("root", new String[]{"a", "b"}); - csConf.setQueues("root.a", new String[]{"a1"}); - csConf.setAutoQueueCreationV2Enabled("root.a.a1", true); + csConf.setQueues(ROOT, new String[]{"a", "b"}); + csConf.setQueues(A, new String[]{"a1"}); + csConf.setAutoQueueCreationV2Enabled(A1, true); cs.reinitialize(csConf, mockRM.getRMContext()); createQueue("root.a.a1.a1-auto"); CSQueue a1Auto = cs.getQueue("root.a.a1.a1-auto"); Assert.assertNotNull("a1-auto should exist", a1Auto); - csConf.setQueues("root", new String[]{"b"}); + csConf.setQueues(ROOT, new String[]{"b"}); cs.reinitialize(csConf, mockRM.getRMContext()); a1Auto = cs.getQueue("root.a.a1.a1-auto"); Assert.assertNull("a1-auto has no parent and should not exist", a1Auto); // Validate dynamic parent deletion - csConf.setState("root.b", QueueState.STOPPED); + csConf.setState(B, QueueState.STOPPED); cs.reinitialize(csConf, mockRM.getRMContext()); - csConf.setAutoQueueCreationV2Enabled("root.b", true); + csConf.setAutoQueueCreationV2Enabled(B, true); cs.reinitialize(csConf, mockRM.getRMContext()); createQueue("root.b.b-auto-parent.b-auto-leaf"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index e333c6a423841..8f2b7f20fc08f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -62,6 +62,7 @@ public class TestCapacitySchedulerNodeLabelUpdate { private final int GB = 1024; private YarnConfiguration conf; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); RMNodeLabelsManager mgr; @@ -79,12 +80,13 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); + conf.setQueues(ROOT, new String[] {"a"}); + conf.setCapacityByLabel(ROOT, "x", 100); + conf.setCapacityByLabel(ROOT, "y", 100); + conf.setCapacityByLabel(ROOT, "z", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + final QueuePath A = new QueuePath(A_PATH); conf.setCapacity(A, 100); conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z")); conf.setCapacityByLabel(A, "x", 100); @@ -100,17 +102,25 @@ private Configuration getConfigurationWithSubQueueLabels( new CapacitySchedulerConfiguration(config); // Define top-level queues - conf2.setQueues(CapacitySchedulerConfiguration.ROOT, + conf2.setQueues(ROOT, new String[] {"a", "b"}); - conf2.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf2.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); - - final String a = CapacitySchedulerConfiguration.ROOT + ".a"; - final String b = CapacitySchedulerConfiguration.ROOT + ".b"; - final String aa1 = a + ".a1"; - final String aa2 = a + ".a2"; - final String aa3 = a + ".a3"; - final String aa4 = a + ".a4"; + conf2.setCapacityByLabel(ROOT, "x", 100); + conf2.setCapacityByLabel(ROOT, "y", 100); + + final String aPath = CapacitySchedulerConfiguration.ROOT + ".a"; + final String bPath = CapacitySchedulerConfiguration.ROOT + ".b"; + final String aa1Path = aPath + ".a1"; + final String aa2Path = aPath + ".a2"; + final String aa3Path = aPath + ".a3"; + final String aa4Path = aPath + ".a4"; + + final QueuePath a = new QueuePath(aPath); + final QueuePath b = new QueuePath(bPath); + final QueuePath aa1 = new QueuePath(aa1Path); + final QueuePath aa2 = new QueuePath(aa2Path); + final QueuePath aa3= new QueuePath(aa3Path); + final QueuePath aa4 = new QueuePath(aa4Path); + conf2.setQueues(a, new String[] {"a1", "a2", "a3", "a4"}); conf2.setCapacity(a, 50); conf2.setCapacity(b, 50); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java index c557354b6f414..332e73f462996 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java @@ -194,8 +194,8 @@ public void testDefaultNodeLabelExpressionQueueConfig() throws Exception { CapacityScheduler cs = new CapacityScheduler(); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); - conf.setDefaultNodeLabelExpression("root.a", " x"); - conf.setDefaultNodeLabelExpression("root.b", " y "); + conf.setDefaultNodeLabelExpression(new QueuePath("root.a"), " x"); + conf.setDefaultNodeLabelExpression(new QueuePath("root.b"), " y "); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); cs.init(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java index b89cae9b7ac0a..e3051165cef31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java @@ -23,15 +23,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerOvercommit; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; - /** * Test changing resources and overcommit in the Capacity Scheduler * {@link CapacityScheduler}. */ public class TestCapacitySchedulerOvercommit extends TestSchedulerOvercommit { + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath DEFAULT = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".default"); @Override protected Configuration getConfiguration() { Configuration conf = super.getConfiguration(); @@ -41,14 +41,10 @@ protected Configuration getConfiguration() { // Remove limits on AMs to allow multiple applications running CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); - csConf.setMaximumApplicationMasterResourcePerQueuePercent( - ROOT_QUEUE_PATH, 100.0f); - csConf.setMaximumAMResourcePercentPerPartition( - CapacitySchedulerConfiguration.ROOT, "", 100.0f); - csConf.setMaximumApplicationMasterResourcePerQueuePercent( - DEFAULT_QUEUE_PATH, 100.0f); - csConf.setMaximumAMResourcePercentPerPartition( - DEFAULT_QUEUE_PATH.getFullPath(), "", 100.0f); + csConf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT, 100.0f); + csConf.setMaximumAMResourcePercentPerPartition(ROOT, "", 100.0f); + csConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT, 100.0f); + csConf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 100.0f); return csConf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java index 73ed20a73665a..24038ebeb8f0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java @@ -62,8 +62,6 @@ import java.util.concurrent.atomic.AtomicLong; import static org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -71,6 +69,9 @@ import static org.mockito.Mockito.when; public class TestCapacitySchedulerPerf { + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath DEFAULT = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".default"); private final int GB = 1024; private String getResourceName(int idx) { @@ -450,9 +451,9 @@ CapacitySchedulerConfiguration createCSConfWithManyQueues(int numQueues) CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(); csconf.setResourceComparator(DominantResourceCalculator.class); - csconf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT_QUEUE_PATH, 100.0f); - csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f); - csconf.setCapacity(DEFAULT_QUEUE_PATH, 0.0f); + csconf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT, 100.0f); + csconf.setMaximumAMResourcePercentPerPartition(ROOT, "", 100.0f); + csconf.setCapacity(DEFAULT, 0.0f); csconf.setOffSwitchPerHeartbeatLimit(numQueues); float capacity = 100.0f / numQueues; @@ -463,13 +464,13 @@ CapacitySchedulerConfiguration createCSConfWithManyQueues(int numQueues) subQueues[i] = queueName; csconf.setMaximumApplicationMasterResourcePerQueuePercent( queuePath, 100.0f); - csconf.setMaximumAMResourcePercentPerPartition(queuePath.getFullPath(), "", 100.0f); + csconf.setMaximumAMResourcePercentPerPartition(queuePath, "", 100.0f); csconf.setCapacity(queuePath, capacity); - csconf.setUserLimitFactor(queuePath.getFullPath(), 100.0f); + csconf.setUserLimitFactor(queuePath, 100.0f); csconf.setMaximumCapacity(queuePath, 100.0f); } - csconf.setQueues("root", subQueues); + csconf.setQueues(ROOT, subQueues); return csconf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java index bb5b790fe2fc5..841b8f9193fba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java @@ -39,13 +39,11 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase { protected Configuration createConfiguration() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { + csConf.setQueues(ROOT, new String[] { QUEUEA, QUEUEB }); - setQueueCapacity(csConf, 50, - CapacitySchedulerConfiguration.ROOT + "." + QUEUEA); - setQueueCapacity(csConf, 50, - CapacitySchedulerConfiguration.ROOT + "." + QUEUEB); + setQueueCapacity(csConf, 50, A_QUEUE_PATH); + setQueueCapacity(csConf, 50, B_QUEUE_PATH); Map aclsOnQueueA = new HashMap(); @@ -54,8 +52,7 @@ protected Configuration createConfiguration() { AccessControlList adminACLonQueueA = new AccessControlList(QUEUE_A_ADMIN); aclsOnQueueA.put(QueueACL.SUBMIT_APPLICATIONS, submitACLonQueueA); aclsOnQueueA.put(QueueACL.ADMINISTER_QUEUE, adminACLonQueueA); - csConf.setAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, - aclsOnQueueA); + csConf.setAcls(A_QUEUE_PATH, aclsOnQueueA); Map aclsOnQueueB = new HashMap(); @@ -64,8 +61,7 @@ protected Configuration createConfiguration() { AccessControlList adminACLonQueueB = new AccessControlList(QUEUE_B_ADMIN); aclsOnQueueB.put(QueueACL.SUBMIT_APPLICATIONS, submitACLonQueueB); aclsOnQueueB.put(QueueACL.ADMINISTER_QUEUE, adminACLonQueueB); - csConf.setAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, - aclsOnQueueB); + csConf.setAcls(B_QUEUE_PATH, aclsOnQueueB); Map aclsOnRootQueue = new HashMap(); @@ -73,7 +69,7 @@ protected Configuration createConfiguration() { AccessControlList adminACLonRoot = new AccessControlList(ROOT_ADMIN); aclsOnRootQueue.put(QueueACL.SUBMIT_APPLICATIONS, submitACLonRoot); aclsOnRootQueue.put(QueueACL.ADMINISTER_QUEUE, adminACLonRoot); - csConf.setAcls(CapacitySchedulerConfiguration.ROOT, aclsOnRootQueue); + csConf.setAcls(ROOT, aclsOnRootQueue); csConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); csConf.set(YarnConfiguration.RM_SCHEDULER, @@ -110,30 +106,30 @@ public void updateConfigWithDAndD1Queues(String rootAcl, String queueDAcl, CapacitySchedulerConfiguration csConf = (CapacitySchedulerConfiguration) getConf(); csConf.clear(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] {QUEUED, QUEUEA, QUEUEB}); String dPath = CapacitySchedulerConfiguration.ROOT + "." + QUEUED; String d1Path = dPath + "." + QUEUED1; - csConf.setQueues(dPath, new String[] {QUEUED1}); - setQueueCapacity(csConf, 100, d1Path); - setQueueCapacity(csConf, 30, CapacitySchedulerConfiguration.ROOT - + "." + QUEUEA); - setQueueCapacity(csConf, 50, CapacitySchedulerConfiguration.ROOT - + "." + QUEUEB); - setQueueCapacity(csConf, 20, dPath); + QueuePath dQueuePath = new QueuePath(dPath); + QueuePath d1QueuePath = new QueuePath(d1Path); + + csConf.setQueues(dQueuePath, new String[] {QUEUED1}); + setQueueCapacity(csConf, 100, d1QueuePath); + setQueueCapacity(csConf, 30, A_QUEUE_PATH); + setQueueCapacity(csConf, 50, B_QUEUE_PATH); + setQueueCapacity(csConf, 20, dQueuePath); if (rootAcl != null) { - setAdminAndSubmitACL(csConf, rootAcl, - CapacitySchedulerConfiguration.ROOT); + setAdminAndSubmitACL(csConf, rootAcl, ROOT); } if (queueDAcl != null) { - setAdminAndSubmitACL(csConf, queueDAcl, dPath); + setAdminAndSubmitACL(csConf, queueDAcl, dQueuePath); } if (queueD1Acl != null) { - setAdminAndSubmitACL(csConf, queueD1Acl, d1Path); + setAdminAndSubmitACL(csConf, queueD1Acl, d1QueuePath); } resourceManager.getResourceScheduler() .reinitialize(csConf, resourceManager.getRMContext()); @@ -141,12 +137,12 @@ public void updateConfigWithDAndD1Queues(String rootAcl, String queueDAcl, private void setQueueCapacity(CapacitySchedulerConfiguration csConf, - float capacity, String queuePath) { + float capacity, QueuePath queuePath) { csConf.setCapacity(queuePath, capacity); } private void setAdminAndSubmitACL(CapacitySchedulerConfiguration csConf, - String queueAcl, String queuePath) { + String queueAcl, QueuePath queuePath) { csConf.setAcl(queuePath, QueueACL.ADMINISTER_QUEUE, queueAcl); csConf.setAcl(queuePath, QueueACL.SUBMIT_APPLICATIONS, queueAcl); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java index a81684a662f07..f58132624a7f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java @@ -43,7 +43,27 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocVcores; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocation; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.unsetMaxAllocation; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1_B1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ExpectedCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.findQueue; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.getDefaultCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfWithoutChildrenOfB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithB1AsParentQueue; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfigurationWithoutB1; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createMockRMContext; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.createResourceManager; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.stopResourceManager; @@ -88,9 +108,9 @@ public void testParseQueue() throws IOException { cs.init(conf); cs.start(); - conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[]{"b1"}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); - conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); + conf.setQueues(A1, new String[]{"b1"}); + conf.setCapacity(A1_B1, 100.0f); + conf.setUserLimitFactor(A1_B1, 100.0f); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), @@ -137,7 +157,8 @@ null, new RMContainerTokenSecretManager(conf), checkQueueStructureCapacities(cs); // Add a new queue b4 - final String b4 = B + ".b4"; + final String b4Path = B + ".b4"; + final QueuePath b4 = new QueuePath(b4Path); final float b4Capacity = 10; final float modifiedB3Capacity = B3_CAPACITY - b4Capacity; @@ -155,15 +176,15 @@ null, new RMContainerTokenSecretManager(conf), final float capB = 20f / 100.0f; Map expectedCapacities = getDefaultCapacities(capA, capB); - expectedCapacities.put(B3, + expectedCapacities.put(B3.getFullPath(), new ExpectedCapacities(modifiedB3Capacity / 100.0f, capB)); - expectedCapacities.put(b4, new ExpectedCapacities(b4Capacity / 100.0f, capB)); + expectedCapacities.put(b4Path, new ExpectedCapacities(b4Capacity / 100.0f, capB)); checkQueueStructureCapacities(cs, expectedCapacities); // Verify parent for B4 CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueB4 = findQueue(queueB, b4); + CSQueue queueB = findQueue(rootQueue, B.getFullPath()); + CSQueue queueB4 = findQueue(queueB, b4Path); assertEquals(queueB, queueB4.getParent()); } finally { @@ -196,12 +217,12 @@ public void testRefreshQueuesMaxAllocationRefresh() throws Exception { ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize()); CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueA = findQueue(rootQueue, A); - CSQueue queueA1 = findQueue(queueA, A1); + CSQueue queueA = findQueue(rootQueue, A.getFullPath()); + CSQueue queueA1 = findQueue(queueA, A1.getFullPath()); assertEquals("queue max allocation", ((LeafQueue) queueA1) .getMaximumAllocation().getMemorySize(), 8192); - setMaxAllocMb(conf, A1_QUEUE_PATH, 4096); + setMaxAllocMb(conf, A1, 4096); try { cs.reinitialize(conf, mockContext); @@ -211,10 +232,10 @@ public void testRefreshQueuesMaxAllocationRefresh() throws Exception { e.getCause().toString().contains("not be decreased")); } - setMaxAllocMb(conf, A1_QUEUE_PATH, 8192); + setMaxAllocMb(conf, A1, 8192); cs.reinitialize(conf, mockContext); - setMaxAllocVcores(conf, A1_QUEUE_PATH, + setMaxAllocVcores(conf, A1, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES - 1); try { cs.reinitialize(conf, mockContext); @@ -236,7 +257,7 @@ public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception { cs.init(conf); cs.start(); // change max allocation for B3 queue to be larger then cluster max - setMaxAllocMb(conf, B3_QUEUE_PATH, + setMaxAllocMb(conf, B3, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 2048); try { cs.reinitialize(conf, mockContext); @@ -246,11 +267,11 @@ public void testRefreshQueuesMaxAllocationPerQueueLarge() throws Exception { e.getCause().getMessage().contains("maximum allocation")); } - setMaxAllocMb(conf, B3_QUEUE_PATH, + setMaxAllocMb(conf, B3, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); cs.reinitialize(conf, mockContext); - setMaxAllocVcores(conf, B3_QUEUE_PATH, + setMaxAllocVcores(conf, B3, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1); try { cs.reinitialize(conf, mockContext); @@ -273,16 +294,16 @@ public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); setMaxAllocVcores(conf, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); - setMaxAllocMb(conf, A1_QUEUE_PATH, 4096); - setMaxAllocVcores(conf, A1_QUEUE_PATH, 2); + setMaxAllocMb(conf, A1, 4096); + setMaxAllocVcores(conf, A1, 2); cs.init(conf); cs.start(); cs.reinitialize(conf, mockContext); checkQueueStructureCapacities(cs); CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueA = findQueue(rootQueue, A); - CSQueue queueA1 = findQueue(queueA, A1); + CSQueue queueA = findQueue(rootQueue, A.getFullPath()); + CSQueue queueA1 = findQueue(queueA, A1.getFullPath()); assertEquals("max capability MB in CS", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, @@ -306,8 +327,8 @@ public void testRefreshQueuesMaxAllocationRefreshLarger() throws Exception { assertEquals("queue max allocation", 4096, queueA1.getMaximumAllocation().getMemorySize()); - setMaxAllocMb(conf, A1_QUEUE_PATH, 6144); - setMaxAllocVcores(conf, A1_QUEUE_PATH, 3); + setMaxAllocMb(conf, A1, 6144); + setMaxAllocVcores(conf, A1, 3); cs.reinitialize(conf, null); // conf will have changed but we shouldn't be able to change max allocation // for the actual queue @@ -344,8 +365,8 @@ public void testRefreshQueuesMaxAllocationCSError() throws Exception { setupQueueConfiguration(conf); setMaxAllocMb(conf, 10240); setMaxAllocVcores(conf, 10); - setMaxAllocMb(conf, A1_QUEUE_PATH, 4096); - setMaxAllocVcores(conf, A1_QUEUE_PATH, 4); + setMaxAllocMb(conf, A1, 4096); + setMaxAllocVcores(conf, A1, 4); cs.init(conf); cs.start(); cs.reinitialize(conf, mockContext); @@ -390,8 +411,8 @@ public void testRefreshQueuesMaxAllocationCSLarger() throws Exception { setupQueueConfiguration(conf); setMaxAllocMb(conf, 10240); setMaxAllocVcores(conf, 10); - setMaxAllocMb(conf, A1_QUEUE_PATH, 4096); - setMaxAllocVcores(conf, A1_QUEUE_PATH, 4); + setMaxAllocMb(conf, A1, 4096); + setMaxAllocVcores(conf, A1, 4); cs.init(conf); cs.start(); cs.reinitialize(conf, mockContext); @@ -403,11 +424,11 @@ public void testRefreshQueuesMaxAllocationCSLarger() throws Exception { cs.getMaximumResourceCapability().getVirtualCores()); CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueA = findQueue(rootQueue, A); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueA1 = findQueue(queueA, A1); - CSQueue queueA2 = findQueue(queueA, A2); - CSQueue queueB2 = findQueue(queueB, B2); + CSQueue queueA = findQueue(rootQueue, A.getFullPath()); + CSQueue queueB = findQueue(rootQueue, B.getFullPath()); + CSQueue queueA1 = findQueue(queueA, A1.getFullPath()); + CSQueue queueA2 = findQueue(queueA, A2.getFullPath()); + CSQueue queueB2 = findQueue(queueB, B2.getFullPath()); assertEquals("queue A1 max allocation MB", 4096, queueA1.getMaximumAllocation().getMemorySize()); @@ -498,8 +519,8 @@ null, new RMContainerTokenSecretManager(conf), + " a queue WITHOUT running apps"); } CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueB3 = findQueue(queueB, B1); + CSQueue queueB = findQueue(rootQueue, B.getFullPath()); + CSQueue queueB3 = findQueue(queueB, B1.getFullPath()); assertNull("Refresh needs to support delete of leaf queue ", queueB3); // reset back to default configuration for testing parent queue delete @@ -549,7 +570,7 @@ null, new RMContainerTokenSecretManager(conf), + " a queue without running apps"); } rootQueue = cs.getRootQueue(); - queueB = findQueue(rootQueue, B); + queueB = findQueue(rootQueue, B.getFullPath()); String message = "Refresh needs to support delete of Parent queue and its children."; assertNull(message, queueB); @@ -624,7 +645,7 @@ null, new RMContainerTokenSecretManager(conf), + " all children of a parent queue(without running apps)."); } CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueB = findQueue(rootQueue, B); + CSQueue queueB = findQueue(rootQueue, B.getFullPath()); assertNotNull("Parent Queue B should not be deleted", queueB); Assert.assertTrue("As Queue'B children are not deleted", queueB instanceof LeafQueue); @@ -715,11 +736,11 @@ public void testQueuesMaxAllocationInheritance() throws Exception { YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); // Test the child queue overrides - setMaxAllocation(conf, ROOT_QUEUE_PATH, + setMaxAllocation(conf, ROOT, "memory-mb=4096,vcores=2"); - setMaxAllocation(conf, A1_QUEUE_PATH, "memory-mb=6144,vcores=2"); - setMaxAllocation(conf, B_QUEUE_PATH, "memory-mb=5120, vcores=2"); - setMaxAllocation(conf, B2_QUEUE_PATH, "memory-mb=1024, vcores=2"); + setMaxAllocation(conf, A1, "memory-mb=6144,vcores=2"); + setMaxAllocation(conf, B, "memory-mb=5120, vcores=2"); + setMaxAllocation(conf, B2, "memory-mb=1024, vcores=2"); cs.init(conf); cs.start(); @@ -727,12 +748,12 @@ public void testQueuesMaxAllocationInheritance() throws Exception { checkQueueStructureCapacities(cs); CSQueue rootQueue = cs.getRootQueue(); - CSQueue queueA = findQueue(rootQueue, A); - CSQueue queueB = findQueue(rootQueue, B); - CSQueue queueA1 = findQueue(queueA, A1); - CSQueue queueA2 = findQueue(queueA, A2); - CSQueue queueB1 = findQueue(queueB, B1); - CSQueue queueB2 = findQueue(queueB, B2); + CSQueue queueA = findQueue(rootQueue, A.getFullPath()); + CSQueue queueB = findQueue(rootQueue, B.getFullPath()); + CSQueue queueA1 = findQueue(queueA, A1.getFullPath()); + CSQueue queueA2 = findQueue(queueA, A2.getFullPath()); + CSQueue queueB1 = findQueue(queueB, B1.getFullPath()); + CSQueue queueB2 = findQueue(queueB, B2.getFullPath()); assertEquals("max capability MB in CS", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, @@ -759,12 +780,12 @@ public void testQueuesMaxAllocationInheritance() throws Exception { queueB2.getMaximumAllocation().getMemorySize()); // Test get the max-allocation from different parent - unsetMaxAllocation(conf, A1_QUEUE_PATH); - unsetMaxAllocation(conf, B_QUEUE_PATH); - unsetMaxAllocation(conf, B1_QUEUE_PATH); - setMaxAllocation(conf, ROOT_QUEUE_PATH, + unsetMaxAllocation(conf, A1); + unsetMaxAllocation(conf, B); + unsetMaxAllocation(conf, B1); + setMaxAllocation(conf, ROOT, "memory-mb=6144,vcores=2"); - setMaxAllocation(conf, A_QUEUE_PATH, "memory-mb=8192,vcores=2"); + setMaxAllocation(conf, A, "memory-mb=8192,vcores=2"); cs.reinitialize(conf, mockContext); @@ -788,9 +809,9 @@ public void testQueuesMaxAllocationInheritance() throws Exception { queueB1.getMaximumAllocation().getVirtualCores()); // Test the default - unsetMaxAllocation(conf, ROOT_QUEUE_PATH); - unsetMaxAllocation(conf, A_QUEUE_PATH); - unsetMaxAllocation(conf, A1_QUEUE_PATH); + unsetMaxAllocation(conf, ROOT); + unsetMaxAllocation(conf, A); + unsetMaxAllocation(conf, A1); cs.reinitialize(conf, mockContext); assertEquals("max capability MB in CS", @@ -837,7 +858,7 @@ public void testVerifyQueuesMaxAllocationConf() throws Exception { cs.reinitialize(conf, mockContext); checkQueueStructureCapacities(cs); - setMaxAllocation(conf, ROOT_QUEUE_PATH, + setMaxAllocation(conf, ROOT, "memory-mb=" + largerMem + ",vcores=2"); try { cs.reinitialize(conf, mockContext); @@ -847,10 +868,10 @@ public void testVerifyQueuesMaxAllocationConf() throws Exception { e.getCause().getMessage().contains("maximum allocation")); } - setMaxAllocation(conf, ROOT_QUEUE_PATH, + setMaxAllocation(conf, ROOT, "memory-mb=4096,vcores=2"); - setMaxAllocation(conf, A_QUEUE_PATH, "memory-mb=6144,vcores=2"); - setMaxAllocation(conf, A1_QUEUE_PATH, "memory-mb=" + largerMem + ",vcores=2"); + setMaxAllocation(conf, A, "memory-mb=6144,vcores=2"); + setMaxAllocation(conf, A1, "memory-mb=" + largerMem + ",vcores=2"); try { cs.reinitialize(conf, mockContext); fail("Queue A1 maximum allocation can't exceed the cluster setting"); @@ -858,7 +879,7 @@ public void testVerifyQueuesMaxAllocationConf() throws Exception { assertTrue("maximum allocation exception", e.getCause().getMessage().contains("maximum allocation")); } - setMaxAllocation(conf, A1_QUEUE_PATH, "memory-mb=8192" + ",vcores=" + largerVcores); + setMaxAllocation(conf, A1, "memory-mb=8192" + ",vcores=" + largerVcores); try { cs.reinitialize(conf, mockContext); fail("Queue A1 maximum allocation can't exceed the cluster setting"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java index ce79528c2599a..ccef92fc156ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java @@ -62,6 +62,17 @@ public class TestCapacitySchedulerSurgicalPreemption extends CapacitySchedulerPreemptionTestBase { private static final int NUM_NM = 5; + + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".a"); + private static final QueuePath B = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".b"); + private static final QueuePath C = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".c"); + private static final QueuePath D = new QueuePath(CapacitySchedulerConfiguration.ROOT + + ".d"); + @Override @Before public void setUp() throws Exception { @@ -350,11 +361,11 @@ public void testPriorityPreemptionWhenAllQueuesAreBelowGuaranteedCapacities() */ conf.setPUOrderingPolicyUnderUtilizedPreemptionEnabled(true); conf.setPUOrderingPolicyUnderUtilizedPreemptionDelay(1000); - conf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT, + conf.setQueueOrderingPolicy(ROOT, CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY); // Queue c has higher priority than a/b - conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1); + conf.setQueuePriority(C, 1); MockRM rm1 = new MockRM(conf); rm1.getRMContext().setNodeLabelManager(mgr); @@ -470,12 +481,12 @@ public void testPriorityPreemptionRequiresMoveReservation() */ conf.setPUOrderingPolicyUnderUtilizedPreemptionEnabled(true); conf.setPUOrderingPolicyUnderUtilizedPreemptionDelay(1000); - conf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT, + conf.setQueueOrderingPolicy(ROOT, CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY); conf.setPUOrderingPolicyUnderUtilizedPreemptionMoveReservation(true); // Queue c has higher priority than a/b - conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1); + conf.setQueuePriority(C, 1); MockRM rm1 = new MockRM(conf); rm1.getRMContext().setNodeLabelManager(mgr); @@ -610,11 +621,11 @@ public void testPriorityPreemptionOnlyTriggeredWhenDemandingQueueUnsatisfied() */ conf.setPUOrderingPolicyUnderUtilizedPreemptionEnabled(true); conf.setPUOrderingPolicyUnderUtilizedPreemptionDelay(1000); - conf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT, + conf.setQueueOrderingPolicy(ROOT, CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY); // Queue c has higher priority than a/b - conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1); + conf.setQueuePriority(C, 1); MockRM rm1 = new MockRM(conf); rm1.getRMContext().setNodeLabelManager(mgr); @@ -750,11 +761,11 @@ public void testPriorityPreemptionFromHighestPriorityQueueAndOldestContainer() */ // A/B has higher priority - conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".a" , 1); - conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".b", 2); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 45f); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".b", 45f); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c", 10f); + conf.setQueuePriority(A , 1); + conf.setQueuePriority(B, 2); + conf.setCapacity(A, 45f); + conf.setCapacity(B, 45f); + conf.setCapacity(C, 10f); testPriorityPreemptionFromHighestPriorityQueueAndOldestContainer(new String[] {"a", "b", "c"}, new String[] {"user", "user", "user"}); @@ -770,7 +781,7 @@ public void testPriorityPreemptionFromHighestPriorityQueueAndOldestContainer() conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); conf.setPUOrderingPolicyUnderUtilizedPreemptionEnabled(true); conf.setPUOrderingPolicyUnderUtilizedPreemptionDelay(1000); - conf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT, + conf.setQueueOrderingPolicy(ROOT, CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY); MockRM rm1 = new MockRM(conf) { @@ -987,12 +998,14 @@ protected RMNodeLabelsManager createNodeLabelManager() { private void initializeConfProperties(CapacitySchedulerConfiguration conf) throws IOException { + QueuePath A_QUEUE_PATH = new QueuePath("root.A"); + QueuePath B_QUEUE_PATH = new QueuePath("root.B"); - conf.setQueues("root", new String[] {"A", "B"}); - conf.setCapacity("root.A", 50); - conf.setCapacity("root.B", 50); - conf.setQueuePriority("root.A", 1); - conf.setQueuePriority("root.B", 2); + conf.setQueues(ROOT, new String[] {"A", "B"}); + conf.setCapacity(A_QUEUE_PATH, 50); + conf.setCapacity(B_QUEUE_PATH, 50); + conf.setQueuePriority(A_QUEUE_PATH, 1); + conf.setQueuePriority(B_QUEUE_PATH, 2); conf.set(PREFIX + "root.ordering-policy", "priority-utilization"); conf.set(PREFIX + "ordering-policy.priority-utilization.underutilized-preemption.enabled", "true"); @@ -1171,11 +1184,11 @@ public void testPreemptionForFragmentatedCluster() throws Exception { this.conf); conf.setLong(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024 * 21); - conf.setQueues("root", new String[] { "a", "b" }); - conf.setCapacity("root.a", 50); - conf.setUserLimitFactor("root.a", 100); - conf.setCapacity("root.b", 50); - conf.setUserLimitFactor("root.b", 100); + conf.setQueues(ROOT, new String[] { "a", "b" }); + conf.setCapacity(A, 50); + conf.setUserLimitFactor(A, 100); + conf.setCapacity(B, 50); + conf.setUserLimitFactor(B, 100); MockRM rm1 = new MockRM(conf); rm1.getRMContext().setNodeLabelManager(mgr); rm1.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java index 45a11ba44aac9..af9d07e40885f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java @@ -54,17 +54,19 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; public class TestCapacitySchedulerWeightMode { - private static final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - private static final String B = CapacitySchedulerConfiguration.ROOT + ".b"; - private static final String A1 = A + ".a1"; - private static final String B1 = B + ".b1"; - private static final String B2 = B + ".b2"; - - private static final QueuePath A_QUEUE_PATH = new QueuePath(A); - private static final QueuePath B_QUEUE_PATH = new QueuePath(B); - private static final QueuePath A1_QUEUE_PATH = new QueuePath(A1); - private static final QueuePath B1_QUEUE_PATH = new QueuePath(B1); - private static final QueuePath B2_QUEUE_PATH = new QueuePath(B2); + private static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default"; + private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final String A1_PATH = A_PATH + ".a1"; + private static final String B1_PATH = B_PATH + ".b1"; + private static final String B2_PATH = B_PATH + ".b2"; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH); + private static final QueuePath A = new QueuePath(A_PATH); + private static final QueuePath B = new QueuePath(B_PATH); + private static final QueuePath A1 = new QueuePath(A1_PATH); + private static final QueuePath B1 = new QueuePath(B1_PATH); + private static final QueuePath B2 = new QueuePath(B2_PATH); private YarnConfiguration conf; @@ -90,7 +92,7 @@ public static CapacitySchedulerConfiguration getConfigWithInheritedAccessibleNod config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] { "a"}); conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 100f); @@ -122,43 +124,43 @@ public static Configuration getCSConfWithQueueLabelsWeightOnly( config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] { "a", "b" }); - conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "x", 100); - conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "y", 100); - conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "z", 100); + conf.setLabeledQueueWeight(ROOT, "x", 100); + conf.setLabeledQueueWeight(ROOT, "y", 100); + conf.setLabeledQueueWeight(ROOT, "z", 100); - conf.setLabeledQueueWeight(A_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(A_QUEUE_PATH, 10); + conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1); + conf.setMaximumCapacity(A, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); - conf.setLabeledQueueWeight(A_QUEUE_PATH, "x", 100); - conf.setLabeledQueueWeight(A_QUEUE_PATH, "y", 50); + conf.setLabeledQueueWeight(A, "x", 100); + conf.setLabeledQueueWeight(A, "y", 50); - conf.setLabeledQueueWeight(B_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 9); - conf.setMaximumCapacity(B_QUEUE_PATH, 100); + conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9); + conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); - conf.setLabeledQueueWeight(B_QUEUE_PATH, "y", 50); - conf.setLabeledQueueWeight(B_QUEUE_PATH, "z", 100); + conf.setLabeledQueueWeight(B, "y", 50); + conf.setLabeledQueueWeight(B, "z", 100); // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); - conf.setLabeledQueueWeight(A1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 100); - conf.setMaximumCapacity(A1_QUEUE_PATH, 100); + conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 100); + conf.setMaximumCapacity(A1, 100); conf.setAccessibleNodeLabels(A1, toSet("x", "y")); conf.setDefaultNodeLabelExpression(A1, "x"); - conf.setLabeledQueueWeight(A1_QUEUE_PATH, "x", 100); - conf.setLabeledQueueWeight(A1_QUEUE_PATH, "y", 100); + conf.setLabeledQueueWeight(A1, "x", 100); + conf.setLabeledQueueWeight(A1, "y", 100); conf.setQueues(B, new String[] { "b1", "b2" }); - conf.setLabeledQueueWeight(B1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 50); - conf.setMaximumCapacity(B1_QUEUE_PATH, 50); + conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 50); + conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); - conf.setLabeledQueueWeight(B2_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 50); - conf.setMaximumCapacity(B2_QUEUE_PATH, 50); + conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 50); + conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); - conf.setLabeledQueueWeight(B2_QUEUE_PATH, "y", 100); - conf.setLabeledQueueWeight(B2_QUEUE_PATH, "z", 100); + conf.setLabeledQueueWeight(B2, "y", 100); + conf.setLabeledQueueWeight(B2, "z", 100); return conf; } @@ -181,28 +183,28 @@ public static Configuration getCSConfWithLabelsParentUseWeightChildUsePct( config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] { "a", "b" }); - conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "x", 100); - conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "y", 100); - conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "z", 100); + conf.setLabeledQueueWeight(ROOT, "x", 100); + conf.setLabeledQueueWeight(ROOT, "y", 100); + conf.setLabeledQueueWeight(ROOT, "z", 100); - conf.setLabeledQueueWeight(A_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(A_QUEUE_PATH, 10); + conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1); + conf.setMaximumCapacity(A, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); - conf.setLabeledQueueWeight(A_QUEUE_PATH, "x", 100); - conf.setLabeledQueueWeight(A_QUEUE_PATH, "y", 50); + conf.setLabeledQueueWeight(A, "x", 100); + conf.setLabeledQueueWeight(A, "y", 50); - conf.setLabeledQueueWeight(B_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 9); - conf.setMaximumCapacity(B_QUEUE_PATH, 100); + conf.setLabeledQueueWeight(B, RMNodeLabelsManager.NO_LABEL, 9); + conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); - conf.setLabeledQueueWeight(B_QUEUE_PATH, "y", 50); - conf.setLabeledQueueWeight(B_QUEUE_PATH, "z", 100); + conf.setLabeledQueueWeight(B, "y", 50); + conf.setLabeledQueueWeight(B, "z", 100); // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); conf.setCapacityByLabel(A1, RMNodeLabelsManager.NO_LABEL, 100); - conf.setMaximumCapacity(A1_QUEUE_PATH, 100); + conf.setMaximumCapacity(A1, 100); conf.setAccessibleNodeLabels(A1, toSet("x", "y")); conf.setDefaultNodeLabelExpression(A1, "x"); conf.setCapacityByLabel(A1, "x", 100); @@ -210,11 +212,11 @@ public static Configuration getCSConfWithLabelsParentUseWeightChildUsePct( conf.setQueues(B, new String[] { "b1", "b2" }); conf.setCapacityByLabel(B1, RMNodeLabelsManager.NO_LABEL, 50); - conf.setMaximumCapacity(B1_QUEUE_PATH, 50); + conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); conf.setCapacityByLabel(B2, RMNodeLabelsManager.NO_LABEL, 50); - conf.setMaximumCapacity(B2_QUEUE_PATH, 50); + conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); conf.setCapacityByLabel(B2, "y", 100); conf.setCapacityByLabel(B2, "z", 100); @@ -240,43 +242,43 @@ public static Configuration getCSConfWithLabelsParentUsePctChildUseWeight( config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] { "a", "b" }); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); + conf.setCapacityByLabel(ROOT, "x", 100); + conf.setCapacityByLabel(ROOT, "y", 100); + conf.setCapacityByLabel(ROOT, "z", 100); conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 10); - conf.setMaximumCapacity(A_QUEUE_PATH, 10); + conf.setMaximumCapacity(A, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); conf.setCapacityByLabel(A, "x", 100); conf.setCapacityByLabel(A, "y", 50); conf.setCapacityByLabel(B, RMNodeLabelsManager.NO_LABEL, 90); - conf.setMaximumCapacity(B_QUEUE_PATH, 100); + conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); conf.setCapacityByLabel(B, "y", 50); conf.setCapacityByLabel(B, "z", 100); // Define 2nd-level queues conf.setQueues(A, new String[] { "a1" }); - conf.setLabeledQueueWeight(A1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(A1_QUEUE_PATH, 100); + conf.setLabeledQueueWeight(A1, RMNodeLabelsManager.NO_LABEL, 1); + conf.setMaximumCapacity(A1, 100); conf.setAccessibleNodeLabels(A1, toSet("x", "y")); conf.setDefaultNodeLabelExpression(A1, "x"); - conf.setLabeledQueueWeight(A1_QUEUE_PATH, "x", 1); - conf.setLabeledQueueWeight(A1_QUEUE_PATH, "y", 1); + conf.setLabeledQueueWeight(A1, "x", 1); + conf.setLabeledQueueWeight(A1, "y", 1); conf.setQueues(B, new String[] { "b1", "b2" }); - conf.setLabeledQueueWeight(B1_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(B1_QUEUE_PATH, 50); + conf.setLabeledQueueWeight(B1, RMNodeLabelsManager.NO_LABEL, 1); + conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); - conf.setLabeledQueueWeight(B2_QUEUE_PATH, RMNodeLabelsManager.NO_LABEL, 1); - conf.setMaximumCapacity(B2_QUEUE_PATH, 50); + conf.setLabeledQueueWeight(B2, RMNodeLabelsManager.NO_LABEL, 1); + conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); - conf.setLabeledQueueWeight(B2_QUEUE_PATH, "y", 1); - conf.setLabeledQueueWeight(B2_QUEUE_PATH, "z", 1); + conf.setLabeledQueueWeight(B2, "y", 1); + conf.setLabeledQueueWeight(B2, "z", 1); return conf; } @@ -330,15 +332,15 @@ public void testGetCapacityOrWeightStringUsingWeights() throws IOException { rm.start(); CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); - String capacityOrWeightString = ((ParentQueue) cs.getQueue(A)) + String capacityOrWeightString = ((ParentQueue) cs.getQueue(A.getFullPath())) .getCapacityOrWeightString(); validateCapacityOrWeightString(capacityOrWeightString, true); - capacityOrWeightString = ((LeafQueue) cs.getQueue(A1)) + capacityOrWeightString = ((LeafQueue) cs.getQueue(A1.getFullPath())) .getCapacityOrWeightString(); validateCapacityOrWeightString(capacityOrWeightString, true); - capacityOrWeightString = ((LeafQueue) cs.getQueue(A1)) + capacityOrWeightString = ((LeafQueue) cs.getQueue(A1.getFullPath())) .getExtendedCapacityOrWeightString(); validateCapacityOrWeightString(capacityOrWeightString, true); } @@ -356,15 +358,15 @@ public void testGetCapacityOrWeightStringParentPctLeafWeights() rm.start(); CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); - String capacityOrWeightString = ((ParentQueue) cs.getQueue(A)) + String capacityOrWeightString = ((ParentQueue) cs.getQueue(A.getFullPath())) .getCapacityOrWeightString(); validateCapacityOrWeightString(capacityOrWeightString, true); - capacityOrWeightString = ((LeafQueue) cs.getQueue(A1)) + capacityOrWeightString = ((LeafQueue) cs.getQueue(A1.getFullPath())) .getCapacityOrWeightString(); validateCapacityOrWeightString(capacityOrWeightString, false); - capacityOrWeightString = ((LeafQueue) cs.getQueue(A1)) + capacityOrWeightString = ((LeafQueue) cs.getQueue(A1.getFullPath())) .getExtendedCapacityOrWeightString(); validateCapacityOrWeightString(capacityOrWeightString, false); } @@ -387,7 +389,7 @@ public void testChildAccessibleNodeLabelsWeightMode() throws Exception { CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler(); - LeafQueue b1 = (LeafQueue) cs.getQueue(B1); + LeafQueue b1 = (LeafQueue) cs.getQueue(B1.getFullPath()); Assert.assertNotNull(b1); Assert.assertTrue(b1.getAccessibleNodeLabels().isEmpty()); @@ -430,11 +432,11 @@ public void testQueueInfoWeight() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration( conf); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] {"a", "b", "default"}); - csConf.setNonLabeledQueueWeight(A_QUEUE_PATH, 1); - csConf.setNonLabeledQueueWeight(B_QUEUE_PATH, 2); - csConf.setNonLabeledQueueWeight(DEFAULT_QUEUE_PATH, 3); + csConf.setNonLabeledQueueWeight(A, 1); + csConf.setNonLabeledQueueWeight(B, 2); + csConf.setNonLabeledQueueWeight(DEFAULT, 3); // Check queue info capacity CapacityScheduler cs = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index ffa939324b97c..deebe3d59be4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -58,8 +58,6 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Assert; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; import static org.junit.Assert.assertEquals; import org.junit.Test; @@ -78,11 +76,13 @@ */ public class TestCapacitySchedulerWithMultiResourceTypes { private static String RESOURCE_1 = "res1"; - - private static final String A_QUEUE = CapacitySchedulerConfiguration.ROOT + ".a"; - private static final String B_QUEUE = CapacitySchedulerConfiguration.ROOT + ".b"; - private static final QueuePath A_QUEUE_PATH = new QueuePath(A_QUEUE); - private static final QueuePath B_QUEUE_PATH = new QueuePath(B_QUEUE); + private static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default"; + private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH); + private static final QueuePath A = new QueuePath(A_PATH); + private static final QueuePath B = new QueuePath(B_PATH); private static float A_CAPACITY = 50.0f; private static float B_CAPACITY = 50.0f; @@ -121,11 +121,11 @@ public void testMaximumAllocationRefreshWithMultipleResourceTypes() throws Excep CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(); - csconf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT_QUEUE_PATH, 100.0f); - csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f); - csconf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, + csconf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT, 100.0f); + csconf.setMaximumAMResourcePercentPerPartition(ROOT, "", 100.0f); + csconf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT, 100.0f); - csconf.setMaximumAMResourcePercentPerPartition("root.default", "", 100.0f); + csconf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 100.0f); csconf.setResourceComparator(DominantResourceCalculator.class); csconf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_1); csconf.setInt(YarnConfiguration.RESOURCE_TYPES + "." + RESOURCE_1 @@ -276,23 +276,22 @@ public void testMaxLimitsOfQueueWithMultipleResources() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT_QUEUE_PATH, 100.0f); - csConf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f); - csConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT_QUEUE_PATH, + csConf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT, 100.0f); + csConf.setMaximumAMResourcePercentPerPartition(ROOT, "", 100.0f); + csConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT, 100.0f); - csConf.setMaximumAMResourcePercentPerPartition("root.default", "", 100.0f); + csConf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 100.0f); csConf.setResourceComparator(DominantResourceCalculator.class); csConf.set(YarnConfiguration.RESOURCE_TYPES, ResourceInformation.GPU_URI); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {"a", "b"}); + csConf.setQueues(ROOT, new String[] {"a", "b"}); // Set each queue to consider 50% each. - csConf.setCapacity(A_QUEUE_PATH, A_CAPACITY); - csConf.setCapacity(B_QUEUE_PATH, B_CAPACITY); - csConf.setMaximumCapacity(A_QUEUE_PATH, 100.0f); - csConf.setUserLimitFactor(A_QUEUE, 2); + csConf.setCapacity(A, A_CAPACITY); + csConf.setCapacity(B, B_CAPACITY); + csConf.setMaximumCapacity(A, 100.0f); + csConf.setUserLimitFactor(A, 2); YarnConfiguration conf = new YarnConfiguration(csConf); // Don't reset resource types since we have already configured resource @@ -416,14 +415,13 @@ public void testConsumeAllExtendedResourcesWithSmallMinUserLimitPct() csconf.setResourceComparator(DominantResourceCalculator.class); // Define top-level queues - csconf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {"a", "b"}); + csconf.setQueues(ROOT, new String[] {"a", "b"}); // Set each queue to contain 50% each. - csconf.setCapacity(A_QUEUE_PATH, A_CAPACITY); - csconf.setCapacity(B_QUEUE_PATH, B_CAPACITY); - csconf.setMaximumCapacity(A_QUEUE_PATH, 100.0f); - csconf.setUserLimitFactor(A_QUEUE, 2); + csconf.setCapacity(A, A_CAPACITY); + csconf.setCapacity(B, B_CAPACITY); + csconf.setMaximumCapacity(A, 100.0f); + csconf.setUserLimitFactor(A, 2); YarnConfiguration yarnConf = new YarnConfiguration(csconf); // Don't reset resource types since we have already configured resource diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java index 442c952ba1211..a212dc60d4c26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java @@ -32,6 +32,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT; import static org.junit.Assert.assertEquals; @@ -59,8 +60,7 @@ public class TestCapacitySchedulerWorkflowPriorityMapping { private static void setWorkFlowPriorityMappings( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues( - CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setQueues(ROOT, new String[] {"a", "b"}); conf.setCapacity(A, A_CAPACITY); conf.setCapacity(B, B_CAPACITY); @@ -76,9 +76,9 @@ private static void setWorkFlowPriorityMappings( conf.setCapacity(B3, B3_CAPACITY); List mappings = Arrays.asList( - new WorkflowPriorityMapping("workflow1", B, Priority.newInstance(2)), - new WorkflowPriorityMapping("workflow2", A1, Priority.newInstance(3)), - new WorkflowPriorityMapping("workflow3", A, Priority.newInstance(4))); + new WorkflowPriorityMapping("workflow1", B.getFullPath(), Priority.newInstance(2)), + new WorkflowPriorityMapping("workflow2", A1.getFullPath(), Priority.newInstance(3)), + new WorkflowPriorityMapping("workflow3", A.getFullPath(), Priority.newInstance(4))); conf.setWorkflowPriorityMappings(mappings); } @@ -100,15 +100,15 @@ public void testWorkflowPriorityMappings() throws Exception { cs.start(); Map> expected = ImmutableMap.of( - A, ImmutableMap.of("workflow3", + A.getFullPath(), ImmutableMap.of("workflow3", new WorkflowPriorityMapping( - "workflow3", A, Priority.newInstance(4))), - B, ImmutableMap.of("workflow1", + "workflow3", A.getFullPath(), Priority.newInstance(4))), + B.getFullPath(), ImmutableMap.of("workflow1", new WorkflowPriorityMapping( - "workflow1", B, Priority.newInstance(2))), - A1, ImmutableMap.of("workflow2", + "workflow1", B.getFullPath(), Priority.newInstance(2))), + A1.getFullPath(), ImmutableMap.of("workflow2", new WorkflowPriorityMapping( - "workflow2", A1, Priority.newInstance(3)))); + "workflow2", A1.getFullPath(), Priority.newInstance(3)))); assertEquals(expected, cs.getWorkflowPriorityMappingsManager() .getWorkflowPriorityMappings()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index 1af3563c52729..8a49087936881 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -204,19 +204,20 @@ private void verifyQueueMetrics(CSQueue queue, private void setupSortedQueues(CapacitySchedulerConfiguration conf) { // Define queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); + final QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + csConf.setQueues(root, new String[] {A, B, C, D}); final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; - conf.setCapacity(Q_A, 25); + conf.setCapacity(new QueuePath(Q_A), 25); final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; - conf.setCapacity(Q_B, 25); + conf.setCapacity(new QueuePath(Q_B), 25); final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; - conf.setCapacity(Q_C, 25); + conf.setCapacity(new QueuePath(Q_C), 25); final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D; - conf.setCapacity(Q_D, 25); + conf.setCapacity(new QueuePath(Q_D), 25); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index ebc9e4e98ff2f..9f43a7d651ae1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -73,13 +73,24 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAX_ASSIGN_PER_HEARTBEAT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; public class TestContainerAllocation { private static final Logger LOG = LoggerFactory .getLogger(TestContainerAllocation.class); - + public static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default"; + public static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + public static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + public static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c"; + public static final String C1_PATH = C_PATH + ".c1"; + public static final String C2_PATH = C_PATH + ".c2"; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH); + private static final QueuePath A = new QueuePath(A_PATH); + private static final QueuePath B = new QueuePath(B_PATH); + private static final QueuePath C = new QueuePath(C_PATH); + private static final QueuePath C1 = new QueuePath(C1_PATH); + private static final QueuePath C2 = new QueuePath(C2_PATH); private final int GB = 1024; private YarnConfiguration conf; @@ -723,7 +734,7 @@ public void testContinuousReservationLookingWhenUsedEqualsMax() throws Exception (CapacitySchedulerConfiguration) TestUtils .getConfigurationWithMultipleQueues(conf); // Set maximum capacity of A to 10 - newConf.setMaximumCapacity(A_QUEUE_PATH, 10); + newConf.setMaximumCapacity(A, 10); MockRM rm1 = new MockRM(newConf); rm1.getRMContext().setNodeLabelManager(mgr); @@ -788,10 +799,8 @@ public void testPendingResourcesConsideringUserLimit() throws Exception { // Set maximum capacity of A to 10 CapacitySchedulerConfiguration newConf = new CapacitySchedulerConfiguration( conf); - newConf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".default", - 0.5f); - newConf.setMaximumAMResourcePercentPerPartition( - CapacitySchedulerConfiguration.ROOT + ".default", "", 1.0f); + newConf.setUserLimitFactor(DEFAULT, 0.5f); + newConf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 1.0f); MockRM rm1 = new MockRM(newConf); rm1.getRMContext().setNodeLabelManager(mgr); @@ -891,14 +900,14 @@ public void testQueuePriorityOrdering() throws Exception { .getConfigurationWithMultipleQueues(conf); // Set ordering policy - newConf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT, + newConf.setQueueOrderingPolicy(ROOT, CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY); // Set maximum capacity of A to 20 - newConf.setMaximumCapacity(A_QUEUE_PATH, 20); - newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1); - newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".b", 2); - newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".a", 3); + newConf.setMaximumCapacity(A, 20); + newConf.setQueuePriority(C, 1); + newConf.setQueuePriority(B, 2); + newConf.setQueuePriority(A, 3); MockRM rm1 = new MockRM(newConf); @@ -1028,7 +1037,7 @@ public void testUserLimitAllocationMultipleContainers() throws Exception { // make sure an unlimited number of containers can be assigned, // overriding the default of 100 after YARN-8896 newConf.set(MAX_ASSIGN_PER_HEARTBEAT, "-1"); - newConf.setUserLimit("root.c", 50); + newConf.setUserLimit(C, 50); MockRM rm1 = new MockRM(newConf); rm1.getRMContext().setNodeLabelManager(mgr); @@ -1091,8 +1100,7 @@ public void testActiveUsersWithOnlyPendingApps() throws Exception { CapacitySchedulerConfiguration newConf = new CapacitySchedulerConfiguration(conf); - newConf.setMaximumAMResourcePercentPerPartition( - CapacitySchedulerConfiguration.ROOT + ".default", "", 0.2f); + newConf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 0.2f); MockRM rm1 = new MockRM(newConf); rm1.getRMContext().setNodeLabelManager(mgr); @@ -1186,7 +1194,7 @@ public void testUnreserveWhenClusterResourceHasEmptyResourceType() newConf .setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false); // Set maximum capacity of queue "a" to 50 - newConf.setMaximumCapacity(A_QUEUE_PATH, 50); + newConf.setMaximumCapacity(A, 50); MockRM rm1 = new MockRM(newConf); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); @@ -1277,10 +1285,10 @@ public void testAllocationCannotBeBlockedWhenFormerQueueReachedItsLimit() .getConfigurationWithMultipleQueues(conf); newConf.setQueues(C, new String[] { "c1", "c2" }); - newConf.setCapacity(C1_QUEUE_PATH, 10); + newConf.setCapacity(C1, 10); newConf - .setMaximumCapacity(C1_QUEUE_PATH, 10); - newConf.setCapacity(C2_QUEUE_PATH, 90); + .setMaximumCapacity(C1, 10); + newConf.setCapacity(C2, 90); newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class, ResourceCalculator.class); @@ -1344,7 +1352,7 @@ public void testContainerRejectionWhenAskBeyondDynamicMax() .getConfigurationWithMultipleQueues(conf); newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, DominantResourceCalculator.class, ResourceCalculator.class); - newConf.set(QueuePrefixes.getQueuePrefix(A_QUEUE_PATH) + newConf.set(QueuePrefixes.getQueuePrefix(A) + MAXIMUM_ALLOCATION_MB, "4096"); MockRM rm1 = new MockRM(newConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index f77cc83dd11bc..3fa5cfeabad53 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -22,11 +22,6 @@ .NO_LABEL; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler .capacity.CSQueueUtils.EPSILON; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler - .capacity.CapacitySchedulerConfiguration.DOT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler - .capacity.CapacitySchedulerConfiguration.ROOT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -241,7 +236,7 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) root = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, - ROOT, + ROOT.getFullPath(), queues, queues, TestUtils.spyHook); queueManager.setRootQueue(root); @@ -269,67 +264,70 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) private static final String C1 = "c1"; private static final String D = "d"; private static final String E = "e"; + final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A_QUEUE_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT + A); private void setupQueueConfiguration( CapacitySchedulerConfiguration conf, - final String newRoot, boolean withNodeLabels) { + final String newRootName, boolean withNodeLabels) { // Define top-level queues - conf.setQueues(ROOT, new String[] {newRoot}); - conf.setMaximumCapacity(ROOT_QUEUE_PATH, 100); + conf.setQueues(ROOT, new String[] {newRootName}); + conf.setMaximumCapacity(ROOT, 100); conf.setAcl(ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); if (withNodeLabels) { - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, LABEL, 100); - conf.setMaximumCapacityByLabel(CapacitySchedulerConfiguration.ROOT, - LABEL, 100); + conf.setCapacityByLabel(ROOT, LABEL, 100); + conf.setMaximumCapacityByLabel(ROOT, LABEL, 100); } - final String Q_newRoot = ROOT + "." + newRoot; - final QueuePath qNewRootPath = new QueuePath(Q_newRoot); - conf.setQueues(Q_newRoot, new String[] {A, B, C, D, E}); - conf.setCapacity(qNewRootPath, 100); - conf.setMaximumCapacity(qNewRootPath, 100); - conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " "); + final String newRootPath = CapacitySchedulerConfiguration.ROOT + "." + newRootName; + final QueuePath newRoot = new QueuePath(newRootPath); + conf.setQueues(newRoot, new String[] {A, B, C, D, E}); + conf.setCapacity(newRoot, 100); + conf.setMaximumCapacity(newRoot, 100); + conf.setAcl(newRoot, QueueACL.SUBMIT_APPLICATIONS, " "); if (withNodeLabels) { - conf.setAccessibleNodeLabels(Q_newRoot, Collections.singleton(LABEL)); - conf.setCapacityByLabel(Q_newRoot, LABEL, 100); - conf.setMaximumCapacityByLabel(Q_newRoot, LABEL, 100); + conf.setAccessibleNodeLabels(newRoot, Collections.singleton(LABEL)); + conf.setCapacityByLabel(newRoot, LABEL, 100); + conf.setMaximumCapacityByLabel(newRoot, LABEL, 100); } - final String Q_A = Q_newRoot + "." + A; - conf.setCapacity(Q_A, 8.5f); - conf.setMaximumCapacity(Q_A, 20); - conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*"); + final QueuePath a = new QueuePath(newRootPath, A); + conf.setCapacity(a, 8.5f); + conf.setMaximumCapacity(a, 20); + conf.setAcl(a, QueueACL.SUBMIT_APPLICATIONS, "*"); if (withNodeLabels) { - conf.setAccessibleNodeLabels(Q_A, Collections.singleton(LABEL)); - conf.setCapacityByLabel(Q_A, LABEL, 100); - conf.setMaximumCapacityByLabel(Q_A, LABEL, 100); + conf.setAccessibleNodeLabels(a, Collections.singleton(LABEL)); + conf.setCapacityByLabel(a, LABEL, 100); + conf.setMaximumCapacityByLabel(a, LABEL, 100); } - - final String Q_B = Q_newRoot + "." + B; - conf.setCapacity(Q_B, 80); - conf.setMaximumCapacity(Q_B, 99); - conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*"); - - final String Q_C = Q_newRoot + "." + C; - conf.setCapacity(Q_C, 1.5f); - conf.setMaximumCapacity(Q_C, 10); - conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " "); - - conf.setQueues(Q_C, new String[] {C1}); - final String Q_C1 = Q_C + "." + C1; - conf.setCapacity(Q_C1, 100); + final QueuePath b = new QueuePath(newRootPath, B); + conf.setCapacity(b, 80); + conf.setMaximumCapacity(b, 99); + conf.setAcl(b, QueueACL.SUBMIT_APPLICATIONS, "*"); - final String Q_D = Q_newRoot + "." + D; - conf.setCapacity(Q_D, 9); - conf.setMaximumCapacity(Q_D, 11); - conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d"); + final String cPath = newRootPath + "." + C; + final QueuePath c = new QueuePath(newRootPath); + conf.setCapacity(c, 1.5f); + conf.setMaximumCapacity(c, 10); + conf.setAcl(c, QueueACL.SUBMIT_APPLICATIONS, " "); - final String Q_E = Q_newRoot + "." + E; - conf.setCapacity(Q_E, 1); - conf.setMaximumCapacity(Q_E, 1); - conf.setAcl(Q_E, QueueACL.SUBMIT_APPLICATIONS, "user_e"); + conf.setQueues(c, new String[] {C1}); + + final QueuePath c1 = new QueuePath(cPath, C1); + conf.setCapacity(c1, 100); + + final QueuePath d = new QueuePath(newRootPath, D); + conf.setCapacity(d, 9); + conf.setMaximumCapacity(d, 11); + conf.setAcl(d, QueueACL.SUBMIT_APPLICATIONS, "user_d"); + + final QueuePath e = new QueuePath(newRootPath, E); + conf.setCapacity(e, 1); + conf.setMaximumCapacity(e, 1); + conf.setAcl(e, QueueACL.SUBMIT_APPLICATIONS, "user_e"); } @@ -482,7 +480,7 @@ public void testPolicyConfiguration() throws Exception { "testPolicyRoot" + System.currentTimeMillis(); OrderingPolicy comPol = - testConf.getAppOrderingPolicy(tproot); + testConf.getAppOrderingPolicy(new QueuePath(tproot)); } @@ -629,8 +627,9 @@ public void testFairConfiguration() throws Exception { CapacitySchedulerConfiguration testConf = new CapacitySchedulerConfiguration(); - String tproot = ROOT + "." + + String tprootPath = ROOT + "." + "testPolicyRoot" + System.currentTimeMillis(); + QueuePath tproot = new QueuePath(tprootPath); OrderingPolicy schedOrder = testConf.getAppOrderingPolicy(tproot); @@ -1095,9 +1094,9 @@ public void testUserLimitCache() throws Exception { when(csContext.getClusterResource()).thenReturn(clusterResource); // working with just one queue - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100); - csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, + csConf.setQueues(ROOT, new String[]{A}); + csConf.setCapacity(A_QUEUE_PATH, 100); + csConf.setMaximumCapacity(A_QUEUE_PATH, 100); queueContext.reinitialize(); @@ -1314,9 +1313,9 @@ public void testUserLimitCacheActiveUsersChanged() throws Exception { when(csContext.getClusterResource()).thenReturn(clusterResource); // working with just one queue - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100); - csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, + csConf.setQueues(ROOT, new String[] {A}); + csConf.setCapacity(A_QUEUE_PATH, 100); + csConf.setMaximumCapacity(A_QUEUE_PATH, 100); queueContext.reinitialize(); @@ -1921,7 +1920,7 @@ public void testUserSpecificUserLimits() throws Exception { // Mock the queue LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); // Set minimum-user-limit-percent for queue "a" in the configs. - csConf.setUserLimit(a.getQueuePath(), 50); + csConf.setUserLimit(a.getQueuePathObject(), 50); // Set weight for "user_0" to be 1.5f for the a queue in the configs. csConf.setFloat("yarn.scheduler.capacity." + a.getQueuePath() + ".user-settings.user_0." + CapacitySchedulerConfiguration.USER_WEIGHT, @@ -3234,7 +3233,7 @@ public void testRackLocalityDelayScheduling() throws Exception { queueContext.reinitialize(); CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(queueContext, - csConf, null, ROOT, newQueues, queues, + csConf, null, ROOT.getFullPath(), newQueues, queues, TestUtils.spyHook); csContext.getCapacitySchedulerQueueManager().setRootQueue(newRoot); root.reinitialize(newRoot, cs.getClusterResource()); @@ -3673,7 +3672,7 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, - ROOT, + ROOT.getFullPath(), newQueues, queues, TestUtils.spyHook); queues = newQueues; @@ -3705,7 +3704,7 @@ public void testLocalityDelaysAfterQueueRefresh() throws Exception { CSQueueStore newQueues = new CSQueueStore(); CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null, - ROOT, + ROOT.getFullPath(), newQueues, queues, TestUtils.spyHook); csContext.getCapacitySchedulerQueueManager().setRootQueue(newRoot); @@ -5136,12 +5135,13 @@ public void testSetupQueueConfigsWithSpecifiedConfiguration() final String leafQueueName = "testSetupQueueConfigsWithSpecifiedConfiguration"; + final QueuePath leafQueuePath = new QueuePath(ROOT.getFullPath(), leafQueueName); assertEquals(0, conf.size()); conf.setNodeLocalityDelay(60); - csConf.setCapacity(ROOT + DOT + leafQueueName, 10); - csConf.setMaximumCapacity(ROOT + DOT + leafQueueName, 100); - csConf.setUserLimitFactor(ROOT + DOT +leafQueueName, 0.1f); + csConf.setCapacity(leafQueuePath, 10); + csConf.setMaximumCapacity(leafQueuePath, 100); + csConf.setUserLimitFactor(leafQueuePath, 0.1f); csConf.setNodeLocalityDelay(30); csConf.setGlobalMaximumApplicationsPerQueue(20); @@ -5158,9 +5158,9 @@ public void testSetupQueueConfigsWithSpecifiedConfiguration() assertEquals(2, leafQueue.getMaxApplicationsPerUser()); //check queue configs - conf.setMaximumAMResourcePercentPerPartition(leafQueue.getQueuePath(), + conf.setMaximumAMResourcePercentPerPartition(leafQueue.getQueuePathObject(), NO_LABEL, 10); - conf.setMaximumCapacity(leafQueue.getQueuePath(), 10); + conf.setMaximumCapacity(leafQueue.getQueuePathObject(), 10); assertEquals(0.1, leafQueue.getMaxAMResourcePerQueuePercent(), EPSILON); @@ -5228,13 +5228,13 @@ public void testMaxApplicationsWithNodeLabels() throws IOException { String rootChild = root.getChildQueues().get(0).getQueuePath(); conf.setCapacityByLabel(ROOT, "test", 100); - conf.setCapacityByLabel(rootChild, "test", 100); - conf.setCapacityByLabel(rootChild + "." + A, "test", 20); - conf.setCapacityByLabel(rootChild + "." + B, "test", 40); - conf.setCapacityByLabel(rootChild + "." + C, "test", 10); - conf.setCapacityByLabel(rootChild + "." + C + "." + C1, "test", 100); - conf.setCapacityByLabel(rootChild + "." + D, "test", 30); - conf.setCapacityByLabel(rootChild + "." + E, "test", 0); + conf.setCapacityByLabel(new QueuePath(rootChild), "test", 100); + conf.setCapacityByLabel(new QueuePath(rootChild, A), "test", 20); + conf.setCapacityByLabel(new QueuePath(rootChild, B), "test", 40); + conf.setCapacityByLabel(new QueuePath(rootChild, C), "test", 10); + conf.setCapacityByLabel(QueuePath.createFromQueues(rootChild, C, C1), "test", 100); + conf.setCapacityByLabel(new QueuePath(rootChild, D), "test", 30); + conf.setCapacityByLabel(new QueuePath(rootChild, E), "test", 0); cs.getCapacitySchedulerQueueManager().reinitConfiguredNodeLabels(conf); cs.setMaxRunningAppsEnforcer(new CSMaxRunningAppsEnforcer(cs)); cs.reinitialize(conf, cs.getRMContext()); @@ -5245,11 +5245,11 @@ public void testMaxApplicationsWithNodeLabels() throws IOException { (int)(conf.getMaximumSystemApplications() * e.getAbsoluteCapacity()), e.getMaxApplications()); - conf.setCapacityByLabel(rootChild + "." + A, "test", 10); - conf.setCapacityByLabel(rootChild + "." + B, "test", 10); - conf.setCapacityByLabel(rootChild + "." + C, "test", 10); - conf.setCapacityByLabel(rootChild + "." + D, "test", 10); - conf.setCapacityByLabel(rootChild + "." + E, "test", 60); + conf.setCapacityByLabel(new QueuePath(rootChild, A), "test", 10); + conf.setCapacityByLabel(new QueuePath(rootChild, B), "test", 10); + conf.setCapacityByLabel(new QueuePath(rootChild, C), "test", 10); + conf.setCapacityByLabel(new QueuePath(rootChild, D), "test", 10); + conf.setCapacityByLabel(new QueuePath(rootChild, E), "test", 60); cs.reinitialize(conf, cs.getRMContext()); e = (LeafQueue) cs.getQueue("e"); @@ -5266,21 +5266,21 @@ public void testRootHasAllNodeLabelsOfItsDescendants() throws IOException { CapacitySchedulerConfiguration conf = csConf; String rootChild = root.getChildQueues().get(0).getQueuePath(); - conf.setCapacityByLabel(rootChild, "test", 100); - conf.setCapacityByLabel(rootChild + "." + A, "test", 20); - conf.setCapacityByLabel(rootChild + "." + B, "test", 40); - conf.setCapacityByLabel(rootChild + "." + C, "test", 10); - conf.setCapacityByLabel(rootChild + "." + C + "." + C1, "test", 100); - conf.setCapacityByLabel(rootChild + "." + D, "test", 30); - conf.setCapacityByLabel(rootChild + "." + E, "test", 0); - - conf.setCapacityByLabel(rootChild, "test2", 100); - conf.setCapacityByLabel(rootChild + "." + A, "test2", 20); - conf.setCapacityByLabel(rootChild + "." + B, "test2", 40); - conf.setCapacityByLabel(rootChild + "." + C, "test2", 10); - conf.setCapacityByLabel(rootChild + "." + C + "." + C1, "test2", 100); - conf.setCapacityByLabel(rootChild + "." + D, "test2", 30); - conf.setCapacityByLabel(rootChild + "." + E, "test2", 0); + conf.setCapacityByLabel(new QueuePath(rootChild), "test", 100); + conf.setCapacityByLabel(new QueuePath(rootChild, A), "test", 20); + conf.setCapacityByLabel(new QueuePath(rootChild, B), "test", 40); + conf.setCapacityByLabel(new QueuePath(rootChild, C), "test", 10); + conf.setCapacityByLabel(QueuePath.createFromQueues(rootChild, C, C1), "test", 100); + conf.setCapacityByLabel(new QueuePath(rootChild, D), "test", 30); + conf.setCapacityByLabel(new QueuePath(rootChild, E), "test", 0); + + conf.setCapacityByLabel(new QueuePath(rootChild), "test2", 100); + conf.setCapacityByLabel(new QueuePath(rootChild, A), "test2", 20); + conf.setCapacityByLabel(new QueuePath(rootChild, B), "test2", 40); + conf.setCapacityByLabel(new QueuePath(rootChild, C), "test2", 10); + conf.setCapacityByLabel(QueuePath.createFromQueues(rootChild, C, C1), "test2", 100); + conf.setCapacityByLabel(new QueuePath(rootChild, D), "test2", 30); + conf.setCapacityByLabel(new QueuePath(rootChild, E), "test2", 0); cs.getCapacitySchedulerQueueManager().reinitConfiguredNodeLabels(conf); cs.setMaxRunningAppsEnforcer(new CSMaxRunningAppsEnforcer(cs)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java index 87aa156f29207..19406c91252fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java @@ -74,6 +74,28 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; public class TestNodeLabelContainerAllocation { + private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c"; + private static final String D_PATH = CapacitySchedulerConfiguration.ROOT + ".d"; + private static final String A1_PATH = A_PATH + ".a1"; + private static final String A2_PATH = A_PATH + ".a2"; + private static final String B1_PATH = B_PATH + ".b1"; + private static final String B2_PATH = B_PATH + ".b2"; + private static final String C1_PATH = C_PATH + ".c1"; + private static final String C2_PATH = C_PATH + ".c2"; + + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A = new QueuePath(A_PATH); + private static final QueuePath B = new QueuePath(B_PATH); + private static final QueuePath C = new QueuePath(C_PATH); + private static final QueuePath D = new QueuePath(D_PATH); + private static final QueuePath A1 = new QueuePath(A1_PATH); + private static final QueuePath A2 = new QueuePath(A2_PATH); + private static final QueuePath B1 = new QueuePath(B1_PATH); + private static final QueuePath B2 = new QueuePath(B2_PATH); + private static final QueuePath C1 = new QueuePath(C1_PATH); + private static final QueuePath C2 = new QueuePath(C2_PATH); private final int GB = 1024; private YarnConfiguration conf; @@ -94,43 +116,37 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); + conf.setQueues(ROOT, new String[] {"a", "b", "c"}); + conf.setCapacityByLabel(ROOT, "x", 100); + conf.setCapacityByLabel(ROOT, "y", 100); + conf.setCapacityByLabel(ROOT, "z", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); conf.setAccessibleNodeLabels(A, toSet("x")); conf.setCapacityByLabel(A, "x", 100); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 20); conf.setAccessibleNodeLabels(B, toSet("y", "z")); conf.setCapacityByLabel(B, "y", 100); conf.setCapacityByLabel(B, "z", 100); - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; conf.setCapacity(C, 70); conf.setMaximumCapacity(C, 70); conf.setAccessibleNodeLabels(C, RMNodeLabelsManager.EMPTY_STRING_SET); - + // Define 2nd-level queues - final String A1 = A + ".a1"; conf.setQueues(A, new String[] {"a1"}); conf.setCapacity(A1, 100); conf.setMaximumCapacity(A1, 100); conf.setCapacityByLabel(A1, "x", 100); - - final String B1 = B + ".b1"; + conf.setQueues(B, new String[] {"b1"}); conf.setCapacity(B1, 100); conf.setMaximumCapacity(B1, 100); conf.setCapacityByLabel(B1, "y", 100); conf.setCapacityByLabel(B1, "z", 100); - final String C1 = C + ".c1"; conf.setQueues(C, new String[] {"c1"}); conf.setCapacity(C1, 100); conf.setMaximumCapacity(C1, 100); @@ -1370,52 +1386,43 @@ public void testPreferenceOfQueuesTowardsNodePartitions() new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + csConf.setQueues(ROOT, new String[] {"a", "b", "c"}); + csConf.setCapacityByLabel(ROOT, "x", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; csConf.setCapacity(A, 33); csConf.setAccessibleNodeLabels(A, toSet("x")); csConf.setCapacityByLabel(A, "x", 33); csConf.setQueues(A, new String[] {"a1", "a2"}); - - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + csConf.setCapacity(B, 33); csConf.setAccessibleNodeLabels(B, toSet("x")); csConf.setCapacityByLabel(B, "x", 33); csConf.setQueues(B, new String[] {"b1", "b2"}); - - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + csConf.setCapacity(C, 34); csConf.setAccessibleNodeLabels(C, toSet("x")); csConf.setCapacityByLabel(C, "x", 34); csConf.setQueues(C, new String[] {"c1", "c2"}); // Define 2nd-level queues - final String A1 = A + ".a1"; csConf.setCapacity(A1, 50); csConf.setCapacityByLabel(A1, "x", 100); csConf.setDefaultNodeLabelExpression(A1, "x"); - - final String A2 = A + ".a2"; + csConf.setCapacity(A2, 50); csConf.setCapacityByLabel(A2, "x", 0); - - final String B1 = B + ".b1"; + csConf.setCapacity(B1, 50); csConf.setCapacityByLabel(B1, "x", 100); csConf.setDefaultNodeLabelExpression(B1, "x"); - - final String B2 = B + ".b2"; + csConf.setCapacity(B2, 50); csConf.setCapacityByLabel(B2, "x", 0); - - final String C1 = C + ".c1"; + csConf.setCapacity(C1, 50); csConf.setCapacityByLabel(C1, "x", 100); csConf.setDefaultNodeLabelExpression(C1, "x"); - - final String C2 = C + ".c2"; + csConf.setCapacity(C2, 50); csConf.setCapacityByLabel(C2, "x", 0); @@ -1563,15 +1570,13 @@ public void testQueuesWithoutAccessUsingPartitionedNodes() throws Exception { new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + csConf.setQueues(ROOT, new String[] {"a", "b"}); + csConf.setCapacityByLabel(ROOT, "x", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; csConf.setCapacity(A, 50); csConf.setAccessibleNodeLabels(A, toSet("x")); csConf.setCapacityByLabel(A, "x", 100); - - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + csConf.setCapacity(B, 50); csConf.setAccessibleNodeLabels(B, new HashSet()); csConf.setUserLimitFactor(B, 5); @@ -1725,18 +1730,16 @@ public RMNodeLabelsManager createNodeLabelManager() { new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", + csConf.setQueues(ROOT, new String[] { "a", "b" }); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + csConf.setCapacityByLabel(ROOT, "x", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; csConf.setCapacity(A, 50); csConf.setAccessibleNodeLabels(A, toSet("x")); csConf.setCapacityByLabel(A, "x", 50); csConf.setMaximumCapacityByLabel(A, "x", 50); csConf.setUserLimit(A, 200); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; csConf.setCapacity(B, 50); csConf.setAccessibleNodeLabels(B, toSet("x")); csConf.setCapacityByLabel(B, "x", 50); @@ -1877,31 +1880,27 @@ public void testQueueUsedCapacitiesUpdate() new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", + csConf.setQueues(ROOT, new String[] { "a", "b" }); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + csConf.setCapacityByLabel(ROOT, "x", 100); /** * Initially, we set A/B's resource 50:50 */ - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; csConf.setCapacity(A, 50); csConf.setAccessibleNodeLabels(A, toSet("x")); csConf.setCapacityByLabel(A, "x", 50); csConf.setQueues(A, new String[] { "a1", "a2" }); - - final String A1 = A + ".a1"; + csConf.setCapacity(A1, 50); csConf.setAccessibleNodeLabels(A1, toSet("x")); csConf.setCapacityByLabel(A1, "x", 50); - - final String A2 = A + ".a2"; + csConf.setCapacity(A2, 50); csConf.setAccessibleNodeLabels(A2, toSet("x")); csConf.setCapacityByLabel(A2, "x", 50); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; csConf.setCapacity(B, 50); csConf.setAccessibleNodeLabels(B, toSet("x")); csConf.setCapacityByLabel(B, "x", 50); @@ -2138,25 +2137,21 @@ public void testOrderOfAllocationOnPartitions() new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", + csConf.setQueues(ROOT, new String[] { "a", "b", "c", "d" }); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + csConf.setCapacityByLabel(ROOT, "x", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; csConf.setCapacity(A, 25); csConf.setAccessibleNodeLabels(A, toSet("x")); csConf.setCapacityByLabel(A, "x", 30); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; csConf.setCapacity(B, 25); csConf.setAccessibleNodeLabels(B, toSet("x")); csConf.setCapacityByLabel(B, "x", 70); - - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + csConf.setAccessibleNodeLabels(C, Collections. emptySet()); csConf.setCapacity(C, 25); - - final String D = CapacitySchedulerConfiguration.ROOT + ".d"; + csConf.setAccessibleNodeLabels(D, Collections. emptySet()); csConf.setCapacity(D, 25); @@ -2340,16 +2335,14 @@ public void testOrderOfAllocationOnPartitionsWhenAccessibilityIsAll() new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] { "a", "b" }); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + csConf.setCapacityByLabel(ROOT, "x", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; csConf.setCapacity(A, 25); csConf.setAccessibleNodeLabels(A, toSet("*")); csConf.setCapacityByLabel(A, "x", 60); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; csConf.setCapacity(B, 75); csConf.setAccessibleNodeLabels(B, toSet("x")); csConf.setCapacityByLabel(B, "x", 40); @@ -2427,17 +2420,15 @@ public void testParentQueueMaxCapsAreRespected() throws Exception { new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", + csConf.setQueues(ROOT, new String[] { "a", "b"}); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + csConf.setCapacityByLabel(ROOT, "x", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; csConf.setCapacity(A, 10); csConf.setAccessibleNodeLabels(A, toSet("x")); csConf.setCapacityByLabel(A, "x", 50); csConf.setMaximumCapacityByLabel(A, "x", 50); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; csConf.setCapacity(B, 90); csConf.setAccessibleNodeLabels(B, toSet("x")); csConf.setCapacityByLabel(B, "x", 50); @@ -2447,14 +2438,12 @@ public void testParentQueueMaxCapsAreRespected() throws Exception { csConf.setQueues(A, new String[] { "a1", "a2"}); - final String A1 = A + ".a1"; csConf.setCapacity(A1, 50); csConf.setAccessibleNodeLabels(A1, toSet("x")); csConf.setCapacityByLabel(A1, "x", 50); csConf.setMaximumCapacityByLabel(A1, "x", 100); csConf.setUserLimitFactor(A1, 100.0f); - final String A2 = A + ".a2"; csConf.setCapacity(A2, 50); csConf.setAccessibleNodeLabels(A2, toSet("x")); csConf.setCapacityByLabel(A2, "x", 50); @@ -2542,20 +2531,19 @@ public void testQueueMetricsWithLabels() throws Exception { this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] { "a", "b" }); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - - final String queueA = CapacitySchedulerConfiguration.ROOT + ".a"; - csConf.setCapacity(queueA, 25); - csConf.setAccessibleNodeLabels(queueA, toSet("x")); - csConf.setCapacityByLabel(queueA, "x", 50); - csConf.setMaximumCapacityByLabel(queueA, "x", 50); - final String queueB = CapacitySchedulerConfiguration.ROOT + ".b"; - csConf.setCapacity(queueB, 75); - csConf.setAccessibleNodeLabels(queueB, toSet("x")); - csConf.setCapacityByLabel(queueB, "x", 50); - csConf.setMaximumCapacityByLabel(queueB, "x", 50); + csConf.setCapacityByLabel(ROOT, "x", 100); + + csConf.setCapacity(A, 25); + csConf.setAccessibleNodeLabels(A, toSet("x")); + csConf.setCapacityByLabel(A, "x", 50); + csConf.setMaximumCapacityByLabel(A, "x", 50); + + csConf.setCapacity(B, 75); + csConf.setAccessibleNodeLabels(B, toSet("x")); + csConf.setCapacityByLabel(B, "x", 50); + csConf.setMaximumCapacityByLabel(B, "x", 50); // set node -> label mgr.addToCluserNodeLabels( @@ -2667,20 +2655,19 @@ public void testQueueMetricsWithLabelsOnDefaultLabelNode() throws Exception { this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] { "a", "b" }); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - - final String queueA = CapacitySchedulerConfiguration.ROOT + ".a"; - csConf.setCapacity(queueA, 25); - csConf.setAccessibleNodeLabels(queueA, toSet("x")); - csConf.setCapacityByLabel(queueA, "x", 50); - csConf.setMaximumCapacityByLabel(queueA, "x", 50); - final String queueB = CapacitySchedulerConfiguration.ROOT + ".b"; - csConf.setCapacity(queueB, 75); - csConf.setAccessibleNodeLabels(queueB, toSet("x")); - csConf.setCapacityByLabel(queueB, "x", 50); - csConf.setMaximumCapacityByLabel(queueB, "x", 50); + csConf.setCapacityByLabel(ROOT, "x", 100); + + csConf.setCapacity(A, 25); + csConf.setAccessibleNodeLabels(A, toSet("x")); + csConf.setCapacityByLabel(A, "x", 50); + csConf.setMaximumCapacityByLabel(A, "x", 50); + + csConf.setCapacity(B, 75); + csConf.setAccessibleNodeLabels(B, toSet("x")); + csConf.setCapacityByLabel(B, "x", 50); + csConf.setMaximumCapacityByLabel(B, "x", 50); csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true); @@ -2948,15 +2935,14 @@ public void testQueueMetricsWithMixedLabels() throws Exception { this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] {"a"}); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + csConf.setCapacityByLabel(ROOT, "x", 100); - final String queueA = CapacitySchedulerConfiguration.ROOT + ".a"; - csConf.setCapacity(queueA, 100); - csConf.setAccessibleNodeLabels(queueA, toSet("x")); - csConf.setCapacityByLabel(queueA, "x", 100); - csConf.setMaximumCapacityByLabel(queueA, "x", 100); + csConf.setCapacity(A, 100); + csConf.setAccessibleNodeLabels(A, toSet("x")); + csConf.setCapacityByLabel(A, "x", 100); + csConf.setMaximumCapacityByLabel(A, "x", 100); // set node -> label // label x exclusivity is set to true @@ -3084,23 +3070,21 @@ public void testTwoLevelQueueMetricsWithLabels() throws Exception { this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] {"a"}); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + csConf.setCapacityByLabel(ROOT, "x", 100); - final String queueA = CapacitySchedulerConfiguration.ROOT + ".a"; - csConf.setCapacity(queueA, 100); - csConf.setAccessibleNodeLabels(queueA, toSet("x")); - csConf.setCapacityByLabel(queueA, "x", 100); - csConf.setMaximumCapacityByLabel(queueA, "x", 100); + csConf.setCapacity(A, 100); + csConf.setAccessibleNodeLabels(A, toSet("x")); + csConf.setCapacityByLabel(A, "x", 100); + csConf.setMaximumCapacityByLabel(A, "x", 100); - csConf.setQueues(queueA, new String[] {"a1"}); - final String queueA1 = queueA + ".a1"; - csConf.setCapacity(queueA1, 100); + csConf.setQueues(A, new String[] {"a1"}); - csConf.setAccessibleNodeLabels(queueA1, toSet("x")); - csConf.setCapacityByLabel(queueA1, "x", 100); - csConf.setMaximumCapacityByLabel(queueA1, "x", 100); + csConf.setCapacity(A1, 100); + csConf.setAccessibleNodeLabels(A1, toSet("x")); + csConf.setCapacityByLabel(A1, "x", 100); + csConf.setMaximumCapacityByLabel(A1, "x", 100); // set node -> label // label x exclusivity is set to true @@ -3304,35 +3288,32 @@ public void testQueueMetricsWithLabelsDisableElasticity() throws Exception { this.conf); // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(ROOT, new String[] { "a", "b" }); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - - final String queueA = CapacitySchedulerConfiguration.ROOT + ".a"; - csConf.setCapacity(queueA, 50); - csConf.setMaximumCapacity(queueA, 100); - csConf.setAccessibleNodeLabels(queueA, toSet("x")); - csConf.setCapacityByLabel(queueA, "x", 50); - csConf.setMaximumCapacityByLabel(queueA, "x", 100); - final String queueB = CapacitySchedulerConfiguration.ROOT + ".b"; - csConf.setCapacity(queueB, 50); - csConf.setMaximumCapacity(queueB, 100); - csConf.setAccessibleNodeLabels(queueB, toSet("x")); - csConf.setCapacityByLabel(queueB, "x", 50); - csConf.setMaximumCapacityByLabel(queueB, "x", 100); + csConf.setCapacityByLabel(ROOT, "x", 100); + + csConf.setCapacity(A, 50); + csConf.setMaximumCapacity(A, 100); + csConf.setAccessibleNodeLabels(A, toSet("x")); + csConf.setCapacityByLabel(A, "x", 50); + csConf.setMaximumCapacityByLabel(A, "x", 100); + + csConf.setCapacity(B, 50); + csConf.setMaximumCapacity(B, 100); + csConf.setAccessibleNodeLabels(B, toSet("x")); + csConf.setCapacityByLabel(B, "x", 50); + csConf.setMaximumCapacityByLabel(B, "x", 100); // Define 2nd-level queues - csConf.setQueues(queueA, new String[] { "a1", + csConf.setQueues(A, new String[] { "a1", "a2"}); - final String A1 = queueA + ".a1"; csConf.setCapacity(A1, 20); csConf.setMaximumCapacity(A1, 60); csConf.setAccessibleNodeLabels(A1, toSet("x")); csConf.setCapacityByLabel(A1, "x", 60); csConf.setMaximumCapacityByLabel(A1, "x", 30); - final String A2 = queueA + ".a2"; csConf.setCapacity(A2, 80); csConf.setMaximumCapacity(A2, 40); csConf.setAccessibleNodeLabels(A2, toSet("x")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 476abc638fba7..0967e9763f9bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -86,7 +86,35 @@ public class TestParentQueue { private final ResourceCalculator resourceComparator = new DefaultResourceCalculator(); - + + private static final String A = "a"; + private static final String B = "b"; + private static final String C = "c"; + private static final String C1 = "c1"; + private static final String C11 = "c11"; + private static final String C111 = "c111"; + private static final String C1111 = "c1111"; + private static final String D = "d"; + private static final String A1 = "a1"; + private static final String A2 = "a2"; + private static final String B1 = "b1"; + private static final String B2 = "b2"; + private static final String B3 = "b3"; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath Q_A = QueuePath.addQueueToPath(ROOT, A); + private static final QueuePath Q_B = QueuePath.addQueueToPath(ROOT, B); + private static final QueuePath Q_C = QueuePath.addQueueToPath(ROOT, C); + private static final QueuePath Q_D = QueuePath.addQueueToPath(ROOT, D); + private static final QueuePath Q_A1 = QueuePath.addQueueToPath(Q_A, A1); + private static final QueuePath Q_A2 = QueuePath.addQueueToPath(Q_A, A2); + private static final QueuePath Q_B1 = QueuePath.addQueueToPath(Q_B, B1); + private static final QueuePath Q_B2 = QueuePath.addQueueToPath(Q_B, B2); + private static final QueuePath Q_B3 = QueuePath.addQueueToPath(Q_B, B3); + private static final QueuePath Q_C1 = QueuePath.addQueueToPath(Q_C, C1); + private static final QueuePath Q_C11 = QueuePath.addQueueToPath(Q_C1, C11); + private static final QueuePath Q_C111 = QueuePath.addQueueToPath(Q_C11, C111); + private static final QueuePath Q_C1111 = QueuePath.addQueueToPath(Q_C111, C1111); + @Before public void setUp() throws Exception { rmContext = TestUtils.getMockRMContext(); @@ -111,17 +139,11 @@ public void setUp() throws Exception { queueContext = new CapacitySchedulerQueueContext(csContext); } - - private static final String A = "a"; - private static final String B = "b"; - private static final String Q_A = - CapacitySchedulerConfiguration.ROOT + "." + A; - private static final String Q_B = - CapacitySchedulerConfiguration.ROOT + "." + B; + private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B}); + conf.setQueues(ROOT, new String[] {A, B}); conf.setCapacity(Q_A, 30); @@ -136,12 +158,12 @@ private void setupSingleLevelQueuesWithAbsoluteResource( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B}); + conf.setQueues(ROOT, new String[]{A, B}); - conf.setMinimumResourceRequirement("", new QueuePath(Q_A), + conf.setMinimumResourceRequirement("", Q_A, QUEUE_A_RESOURCE); - conf.setMinimumResourceRequirement("", new QueuePath(Q_B), + conf.setMinimumResourceRequirement("", Q_B, QUEUE_B_RESOURCE); queueContext.reinitialize(); @@ -427,60 +449,40 @@ public void testSingleLevelQueuesPrecision() throws Exception { } } - private static final String C = "c"; - private static final String C1 = "c1"; - private static final String C11 = "c11"; - private static final String C111 = "c111"; - private static final String C1111 = "c1111"; - - private static final String D = "d"; - private static final String A1 = "a1"; - private static final String A2 = "a2"; - private static final String B1 = "b1"; - private static final String B2 = "b2"; - private static final String B3 = "b3"; - private static final String B4 = "b4"; - private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) { // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); + csConf.setQueues(ROOT, new String[] {A, B, C, D}); conf.setCapacity(Q_A, 10); conf.setCapacity(Q_B, 50); - - final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; + conf.setCapacity(Q_C, 19.5f); - - final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D; + conf.setCapacity(Q_D, 20.5f); // Define 2-nd level queues conf.setQueues(Q_A, new String[] {A1, A2}); - conf.setCapacity(Q_A + "." + A1, 50); - conf.setCapacity(Q_A + "." + A2, 50); + conf.setCapacity(Q_A1, 50); + conf.setCapacity(Q_A2, 50); conf.setQueues(Q_B, new String[] {B1, B2, B3}); - conf.setCapacity(Q_B + "." + B1, 10); - conf.setCapacity(Q_B + "." + B2, 20); - conf.setCapacity(Q_B + "." + B3, 70); + conf.setCapacity(Q_B1, 10); + conf.setCapacity(Q_B2, 20); + conf.setCapacity(Q_B3, 70); conf.setQueues(Q_C, new String[] {C1}); - final String Q_C1= Q_C + "." + C1; conf.setCapacity(Q_C1, 100); conf.setQueues(Q_C1, new String[] {C11}); - final String Q_C11= Q_C1 + "." + C11; conf.setCapacity(Q_C11, 100); conf.setQueues(Q_C11, new String[] {C111}); - final String Q_C111= Q_C11 + "." + C111; conf.setCapacity(Q_C111, 100); //Leaf Queue conf.setQueues(Q_C111, new String[] {C1111}); - final String Q_C1111= Q_C111 + "." + C1111; conf.setCapacity(Q_C1111, 100); queueContext.reinitialize(); } @@ -667,9 +669,9 @@ public void testQueueCapacitySettingChildZero() throws Exception { setupMultiLevelQueues(csConf); // set child queues capacity to 0 when parents not 0 - csConf.setCapacity(Q_B + "." + B1, 0); - csConf.setCapacity(Q_B + "." + B2, 0); - csConf.setCapacity(Q_B + "." + B3, 0); + csConf.setCapacity(Q_B1, 0); + csConf.setCapacity(Q_B2, 0); + csConf.setCapacity(Q_B3, 0); queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); @@ -722,9 +724,9 @@ public void testQueueCapacitySettingParentZeroChildren50pctZeroSumAllowed() // and allow zero capacity sum csConf.setCapacity(Q_B, 0); csConf.setCapacity(Q_A, 100); - csConf.setCapacity(Q_B + "." + B1, 10); - csConf.setCapacity(Q_B + "." + B2, 20); - csConf.setCapacity(Q_B + "." + B3, 20); + csConf.setCapacity(Q_B1, 10); + csConf.setCapacity(Q_B2, 20); + csConf.setCapacity(Q_B3, 20); csConf.setAllowZeroCapacitySum(Q_B, true); queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); @@ -743,9 +745,9 @@ public void testQueueCapacitySettingParentNonZeroChildrenZeroSumAllowed() // and allow zero capacity sum csConf.setCapacity(Q_B, 10); csConf.setCapacity(Q_A, 50); - csConf.setCapacity(Q_B + "." + B1, 0); - csConf.setCapacity(Q_B + "." + B2, 0); - csConf.setCapacity(Q_B + "." + B3, 0); + csConf.setCapacity(Q_B1, 0); + csConf.setCapacity(Q_B2, 0); + csConf.setCapacity(Q_B3, 0); csConf.setAllowZeroCapacitySum(Q_B, true); queueContext.reinitialize(); CSQueueStore queues = new CSQueueStore(); @@ -761,9 +763,9 @@ public void testQueueCapacityZero() throws Exception { // set parent and child capacity to 0 csConf.setCapacity(Q_B, 0); - csConf.setCapacity(Q_B + "." + B1, 0); - csConf.setCapacity(Q_B + "." + B2, 0); - csConf.setCapacity(Q_B + "." + B3, 0); + csConf.setCapacity(Q_B1, 0); + csConf.setCapacity(Q_B2, 0); + csConf.setCapacity(Q_B3, 0); csConf.setCapacity(Q_A, 60); queueContext.reinitialize(); @@ -959,12 +961,10 @@ public boolean hasQueueACL(List aclInfos, QueueACL acl, String public void testQueueAcl() throws Exception { setupMultiLevelQueues(csConf); - csConf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); - csConf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.ADMINISTER_QUEUE, " "); + csConf.setAcl(ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); + csConf.setAcl(ROOT, QueueACL.ADMINISTER_QUEUE, " "); - final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; csConf.setAcl(Q_C, QueueACL.ADMINISTER_QUEUE, "*"); - final String Q_C11= Q_C + "." + C1 + "." + C11; csConf.setAcl(Q_C11, QueueACL.SUBMIT_APPLICATIONS, "*"); queueContext.reinitialize(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java index 695ceb782de88..a49f155b87d06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java @@ -42,6 +42,9 @@ public class TestQueueMappings { CapacitySchedulerConfiguration.ROOT + "." + Q1; private final static String Q2_PATH = CapacitySchedulerConfiguration.ROOT + "." + Q2; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath Q1_QUEUE_PATH = new QueuePath(Q1_PATH); + private static final QueuePath Q2_QUEUE_PATH = new QueuePath(Q2_PATH); private CapacityScheduler cs; private YarnConfiguration conf; @@ -63,10 +66,10 @@ public void setup() { private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { Q1, Q2 }); + conf.setQueues(ROOT, new String[] { Q1, Q2 }); - conf.setCapacity(Q1_PATH, 10); - conf.setCapacity(Q2_PATH, 90); + conf.setCapacity(Q1_QUEUE_PATH, 10); + conf.setCapacity(Q2_QUEUE_PATH, 90); LOG.info("Setup top-level queues q1 and q2"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index f9696ee41e0b5..22414e80b2d47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -45,14 +45,50 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_QUEUE_PATH; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT_QUEUE_PATH; - public class TestQueueParsing { private static final Logger LOG = LoggerFactory.getLogger(TestQueueParsing.class); - + private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c"; + private static final String A1_PATH = A_PATH + ".a1"; + private static final String A2_PATH = A_PATH + ".a2"; + private static final String B1_PATH = B_PATH + ".b1"; + private static final String B2_PATH = B_PATH + ".b2"; + private static final String B3_PATH = B_PATH + ".b3"; + private static final String B4_PATH = B_PATH + ".b4"; + private static final String C1_PATH = C_PATH + ".c1"; + private static final String C2_PATH = C_PATH + ".c2"; + private static final String C3_PATH = C_PATH + ".c3"; + private static final String C4_PATH = C_PATH + ".c4"; + private static final String C11_PATH = C1_PATH + ".c11"; + private static final String C12_PATH = C1_PATH + ".c12"; + private static final String C13_PATH = C1_PATH + ".c13"; + private static final String AX_PATH = "root.a.x"; + private static final String AY_PATH = "root.a.y"; + private static final String X_PATH = "root.x"; + + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A = new QueuePath(A_PATH); + private static final QueuePath B = new QueuePath(B_PATH); + private static final QueuePath C = new QueuePath(C_PATH); + private static final QueuePath A1 = new QueuePath(A1_PATH); + private static final QueuePath A2 = new QueuePath(A2_PATH); + private static final QueuePath B1 = new QueuePath(B1_PATH); + private static final QueuePath B2 = new QueuePath(B2_PATH); + private static final QueuePath B3 = new QueuePath(B3_PATH); + private static final QueuePath B4 = new QueuePath(B4_PATH); + private static final QueuePath C1 = new QueuePath(C1_PATH); + private static final QueuePath C2 = new QueuePath(C2_PATH); + private static final QueuePath C3 = new QueuePath(C3_PATH); + private static final QueuePath C4 = new QueuePath(C4_PATH); + private static final QueuePath C11 = new QueuePath(C11_PATH); + private static final QueuePath C12 = new QueuePath(C12_PATH); + private static final QueuePath C13 = new QueuePath(C13_PATH); + private static final QueuePath AX = new QueuePath(AX_PATH); + private static final QueuePath AY = new QueuePath(AY_PATH); + private static final QueuePath X = new QueuePath(X_PATH); private static final double DELTA = 0.000001; private RMNodeLabelsManager nodeLabelManager; @@ -99,17 +135,14 @@ private void setupQueueConfigurationWithSpacesShouldBeTrimmed( // Define top-level queues conf.set( QueuePrefixes - .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) + .getQueuePrefix(ROOT) + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setCapacity(B, 20); - - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + conf.setCapacity(C, 70); conf.setMaximumCapacity(C, 70); } @@ -119,17 +152,14 @@ private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed( // Define top-level queues conf.set( QueuePrefixes - .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) + .getQueuePrefix(ROOT) + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 20); - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; conf.setCapacity(C, 70); conf.setMaximumCapacity(C, 70); @@ -137,43 +167,33 @@ private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed( conf.set(QueuePrefixes.getQueuePrefix(A) + CapacitySchedulerConfiguration.QUEUES, "a1, a2 "); - final String A1 = CapacitySchedulerConfiguration.ROOT + ".a.a1"; conf.setCapacity(A1, 60); - final String A2 = CapacitySchedulerConfiguration.ROOT + ".a.a2"; conf.setCapacity(A2, 40); } private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); + conf.setQueues(ROOT, new String[] {"a", "b", "c"}); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setCapacity(B, 20); - - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + conf.setCapacity(C, 70); conf.setMaximumCapacity(C, 70); LOG.info("Setup top-level queues"); // Define 2nd-level queues - final String A1 = A + ".a1"; - final String A2 = A + ".a2"; conf.setQueues(A, new String[] {"a1", "a2"}); conf.setCapacity(A1, 30); conf.setMaximumCapacity(A1, 45); conf.setCapacity(A2, 70); conf.setMaximumCapacity(A2, 85); - - final String B1 = B + ".b1"; - final String B2 = B + ".b2"; - final String B3 = B + ".b3"; + conf.setQueues(B, new String[] {"b1", "b2", "b3"}); conf.setCapacity(B1, 50); conf.setMaximumCapacity(B1, 85); @@ -182,10 +202,6 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { conf.setCapacity(B3, 20); conf.setMaximumCapacity(B3, 35); - final String C1 = C + ".c1"; - final String C2 = C + ".c2"; - final String C3 = C + ".c3"; - final String C4 = C + ".c4"; conf.setQueues(C, new String[] {"c1", "c2", "c3", "c4"}); conf.setCapacity(C1, 50); conf.setMaximumCapacity(C1, 55); @@ -199,9 +215,6 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { LOG.info("Setup 2nd-level queues"); // Define 3rd-level queues - final String C11 = C1 + ".c11"; - final String C12 = C1 + ".c12"; - final String C13 = C1 + ".c13"; conf.setQueues(C1, new String[] {"c11", "c12", "c13"}); conf.setCapacity(C11, 15); conf.setMaximumCapacity(C11, 30); @@ -218,7 +231,7 @@ public void testRootQueueParsing() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); // non-100 percent value will throw IllegalArgumentException - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 90); + conf.setCapacity(ROOT, 90); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(new YarnConfiguration()); @@ -231,13 +244,11 @@ public void testRootQueueParsing() throws Exception { public void testMaxCapacity() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); + conf.setQueues(ROOT, new String[] {"a", "b", "c"}); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 50); conf.setMaximumCapacity(A, 60); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 50); conf.setMaximumCapacity(B, 45); // Should throw an exception @@ -267,7 +278,7 @@ public void testMaxCapacity() throws Exception { fail = false; try { - LeafQueue a = (LeafQueue)capacityScheduler.getQueue(A); + LeafQueue a = (LeafQueue)capacityScheduler.getQueue(A_PATH); a.setMaxCapacity(45); } catch (IllegalArgumentException iae) { fail = true; @@ -279,29 +290,22 @@ public void testMaxCapacity() throws Exception { private void setupQueueConfigurationWithoutLabels(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setQueues(ROOT, new String[] {"a", "b"}); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setCapacity(B, 90); LOG.info("Setup top-level queues"); // Define 2nd-level queues - final String A1 = A + ".a1"; - final String A2 = A + ".a2"; conf.setQueues(A, new String[] {"a1", "a2"}); conf.setCapacity(A1, 30); conf.setMaximumCapacity(A1, 45); conf.setCapacity(A2, 70); conf.setMaximumCapacity(A2, 85); - - final String B1 = B + ".b1"; - final String B2 = B + ".b2"; - final String B3 = B + ".b3"; + conf.setQueues(B, new String[] {"b1", "b2", "b3"}); conf.setCapacity(B1, 50); conf.setMaximumCapacity(B1, 85); @@ -313,22 +317,18 @@ private void setupQueueConfigurationWithoutLabels(CapacitySchedulerConfiguration private void setupQueueConfigurationWithLabels(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100); + conf.setQueues(ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(ROOT, "red", 100); + conf.setCapacityByLabel(ROOT, "blue", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); - - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setCapacity(B, 90); LOG.info("Setup top-level queues"); // Define 2nd-level queues - final String A1 = A + ".a1"; - final String A2 = A + ".a2"; conf.setQueues(A, new String[] {"a1", "a2"}); conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue")); conf.setCapacityByLabel(A, "red", 50); @@ -345,10 +345,7 @@ private void setupQueueConfigurationWithLabels(CapacitySchedulerConfiguration co conf.setAccessibleNodeLabels(A2, ImmutableSet.of("red")); conf.setCapacityByLabel(A2, "red", 50); conf.setMaximumCapacityByLabel(A2, "red", 60); - - final String B1 = B + ".b1"; - final String B2 = B + ".b2"; - final String B3 = B + ".b3"; + conf.setQueues(B, new String[] {"b1", "b2", "b3"}); conf.setAccessibleNodeLabels(B, ImmutableSet.of("red", "blue")); conf.setCapacityByLabel(B, "red", 50); @@ -373,24 +370,20 @@ private void setupQueueConfigurationWithLabels(CapacitySchedulerConfiguration co private void setupQueueConfigurationWithLabelsAndReleaseCheck (CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100); + conf.setQueues(ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(ROOT, "red", 100); + conf.setCapacityByLabel(ROOT, "blue", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; // The cap <= max-cap check is not needed conf.setCapacity(A, 50); conf.setMaximumCapacity(A, 100); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 50); conf.setMaximumCapacity(B, 100); LOG.info("Setup top-level queues"); // Define 2nd-level queues - final String A1 = A + ".a1"; - final String A2 = A + ".a2"; conf.setQueues(A, new String[] {"a1", "a2"}); conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue")); conf.setCapacityByLabel(A, "red", 50); @@ -411,9 +404,6 @@ private void setupQueueConfigurationWithLabels(CapacitySchedulerConfiguration co conf.setCapacityByLabel(A2, "red", 40); conf.setMaximumCapacityByLabel(A2, "red", 60); - final String B1 = B + ".b1"; - final String B2 = B + ".b2"; - final String B3 = B + ".b3"; conf.setQueues(B, new String[] {"b1", "b2", "b3"}); conf.setAccessibleNodeLabels(B, ImmutableSet.of("red", "blue")); conf.setCapacityByLabel(B, "red", 50); @@ -442,12 +432,11 @@ private void setupQueueConfigurationWithLabels(CapacitySchedulerConfiguration co private void setupQueueConfigurationWithLabelsInherit( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100); + conf.setQueues(ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(ROOT, "red", 100); + conf.setCapacityByLabel(ROOT, "blue", 100); // Set A configuration - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); conf.setQueues(A, new String[] {"a1", "a2"}); @@ -456,14 +445,10 @@ private void setupQueueConfigurationWithLabelsInherit( conf.setCapacityByLabel(A, "blue", 100); // Set B configuraiton - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 90); conf.setAccessibleNodeLabels(B, CommonNodeLabelsManager.EMPTY_STRING_SET); // Define 2nd-level queues - final String A1 = A + ".a1"; - final String A2 = A + ".a2"; - conf.setCapacity(A1, 30); conf.setMaximumCapacity(A1, 45); conf.setCapacityByLabel(A1, "red", 50); @@ -478,10 +463,9 @@ private void setupQueueConfigurationWithLabelsInherit( private void setupQueueConfigurationWithSingleLevel( CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setQueues(ROOT, new String[] {"a", "b"}); // Set A configuration - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue")); @@ -489,7 +473,6 @@ private void setupQueueConfigurationWithSingleLevel( conf.setCapacityByLabel(A, "blue", 90); // Set B configuraiton - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 90); conf.setAccessibleNodeLabels(B, ImmutableSet.of("red", "blue")); conf.setCapacityByLabel(B, "red", 10); @@ -843,7 +826,7 @@ public void testQueueParsingWithUnusedLabels() throws IOException { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); - csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT, labels); + csConf.setAccessibleNodeLabels(ROOT, labels); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); @@ -954,7 +937,7 @@ public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfiguration(csConf); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5); + csConf.setCapacity(C2, 5); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = @@ -984,8 +967,7 @@ public void testQueueParsingFailWhenSumOfChildrenLabeledCapacityNot100Percent() CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3", - "red", 24); + csConf.setCapacityByLabel(B3, "red", 24); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = @@ -1015,12 +997,9 @@ public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); - csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3", - "red", 24); - csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT, - ImmutableSet.of(RMNodeLabelsManager.ANY)); - csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b", - ImmutableSet.of(RMNodeLabelsManager.ANY)); + csConf.setCapacityByLabel(B3, "red", 24); + csConf.setAccessibleNodeLabels(ROOT, ImmutableSet.of(RMNodeLabelsManager.ANY)); + csConf.setAccessibleNodeLabels(B, ImmutableSet.of(RMNodeLabelsManager.ANY)); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = @@ -1042,11 +1021,11 @@ public void testQueueParsingWithMoveQueue() YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); - csConf.setQueues("root", new String[] { "a" }); - csConf.setQueues("root.a", new String[] { "x", "y" }); - csConf.setCapacity("root.a", 100); - csConf.setCapacity("root.a.x", 50); - csConf.setCapacity("root.a.y", 50); + csConf.setQueues(ROOT, new String[] { "a" }); + csConf.setQueues(A, new String[] { "x", "y" }); + csConf.setCapacity(A, 100); + csConf.setCapacity(AX, 50); + csConf.setCapacity(AY, 50); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = @@ -1060,11 +1039,11 @@ public void testQueueParsingWithMoveQueue() capacityScheduler.init(csConf); capacityScheduler.start(); - csConf.setQueues("root", new String[] { "a", "x" }); - csConf.setQueues("root.a", new String[] { "y" }); - csConf.setCapacity("root.x", 50); - csConf.setCapacity("root.a", 50); - csConf.setCapacity("root.a.y", 100); + csConf.setQueues(ROOT, new String[] { "a", "x" }); + csConf.setQueues(A, new String[] { "y" }); + csConf.setCapacity(X, 50); + csConf.setCapacity(A, 50); + csConf.setCapacity(AY, 100); capacityScheduler.reinitialize(csConf, rmContext); } @@ -1080,12 +1059,11 @@ public void testRMStartWrongNodeCapacity() throws Exception { new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a" }); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); + conf.setQueues(ROOT, new String[] { "a" }); + conf.setCapacityByLabel(ROOT, "x", 100); + conf.setCapacityByLabel(ROOT, "y", 100); + conf.setCapacityByLabel(ROOT, "z", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 100); conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z")); conf.setCapacityByLabel(A, "x", 100); @@ -1126,9 +1104,9 @@ public void testQueueOrderingPolicyUpdatedAfterReinitialize() capacityScheduler.start(); // Add a new b4 queue - csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".b", + csConf.setQueues(B, new String[] { "b1", "b2", "b3", "b4" }); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".b.b4", 0f); + csConf.setCapacity(B4, 0f); ParentQueue bQ = (ParentQueue) capacityScheduler.getQueue("b"); checkEqualsToQueueSet(bQ.getChildQueues(), new String[] { "b1", "b2", "b3" }); @@ -1157,17 +1135,16 @@ public void testQueueCapacityWithWeight() throws Exception { new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a" }); - conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "x", 100); - conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "y", 100); - conf.setLabeledQueueWeight(ROOT_QUEUE_PATH, "z", 100); + conf.setQueues(ROOT, new String[] { "a" }); + conf.setLabeledQueueWeight(ROOT, "x", 100); + conf.setLabeledQueueWeight(ROOT, "y", 100); + conf.setLabeledQueueWeight(ROOT, "z", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - conf.setNonLabeledQueueWeight(A_QUEUE_PATH, 100); + conf.setNonLabeledQueueWeight(A, 100); conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z")); - conf.setLabeledQueueWeight(A_QUEUE_PATH, "x", 100); - conf.setLabeledQueueWeight(A_QUEUE_PATH, "y", 100); - conf.setLabeledQueueWeight(A_QUEUE_PATH, "z", 70); + conf.setLabeledQueueWeight(A, "x", 100); + conf.setLabeledQueueWeight(A, "y", 100); + conf.setLabeledQueueWeight(A, "z", 70); MockRM rm = null; try { rm = new MockRM(conf) { @@ -1185,10 +1162,10 @@ public RMNodeLabelsManager createNodeLabelManager() { verifyQueueAbsCapacity(rm, CapacitySchedulerConfiguration.ROOT, "y", 1f); verifyQueueAbsCapacity(rm, CapacitySchedulerConfiguration.ROOT, "z", 1f); - verifyQueueAbsCapacity(rm, A, "", 1f); - verifyQueueAbsCapacity(rm, A, "x", 1f); - verifyQueueAbsCapacity(rm, A, "y", 1f); - verifyQueueAbsCapacity(rm, A, "z", 1f); + verifyQueueAbsCapacity(rm, A_PATH, "", 1f); + verifyQueueAbsCapacity(rm, A_PATH, "x", 1f); + verifyQueueAbsCapacity(rm, A_PATH, "y", 1f); + verifyQueueAbsCapacity(rm, A_PATH, "z", 1f); } @Test @@ -1197,21 +1174,19 @@ public void testQueueParsingWithDefaultUserLimitValues() YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); - final String queueA = CapacitySchedulerConfiguration.ROOT + ".a"; - final String queueB = CapacitySchedulerConfiguration.ROOT + ".b"; // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + csConf.setQueues(ROOT, new String[] {"a", "b"}); // Set default value csConf.setDefaultUserLimit(20); csConf.setDefaultUserLimitFactor(2.0f); // Set A configuration and let B use default values - csConf.setCapacity(queueA, 50); - csConf.setUserLimit(queueA, 15); - csConf.setUserLimitFactor(queueA, 1.5f); - csConf.setCapacity(queueB, 50); + csConf.setCapacity(A, 50); + csConf.setUserLimit(A, 15); + csConf.setUserLimitFactor(A, 1.5f); + csConf.setCapacity(B, 50); // Test CapacityScheduler capacityScheduler = new CapacityScheduler(); @@ -1226,23 +1201,23 @@ public void testQueueParsingWithDefaultUserLimitValues() capacityScheduler.init(csConf); capacityScheduler.start(); Assert.assertEquals(15, - ((LeafQueue)capacityScheduler.getQueue(queueA)).getUserLimit(), DELTA); + ((LeafQueue)capacityScheduler.getQueue(A_PATH)).getUserLimit(), DELTA); Assert.assertEquals(1.5, - ((LeafQueue)capacityScheduler.getQueue(queueA)).getUserLimitFactor(), DELTA); + ((LeafQueue)capacityScheduler.getQueue(A_PATH)).getUserLimitFactor(), DELTA); Assert.assertEquals(20, - ((LeafQueue)capacityScheduler.getQueue(queueB)).getUserLimit(), DELTA); + ((LeafQueue)capacityScheduler.getQueue(B_PATH)).getUserLimit(), DELTA); Assert.assertEquals(2.0, - ((LeafQueue)capacityScheduler.getQueue(queueB)).getUserLimitFactor(), DELTA); + ((LeafQueue)capacityScheduler.getQueue(B_PATH)).getUserLimitFactor(), DELTA); ServiceOperations.stopQuietly(capacityScheduler); // Use hadoop default value conf = new YarnConfiguration(); csConf = new CapacitySchedulerConfiguration(conf); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - csConf.setCapacity(queueA, 50); - csConf.setUserLimit(queueA, 15); - csConf.setUserLimitFactor(queueA, 1.5f); - csConf.setCapacity(queueB, 50); + csConf.setQueues(ROOT, new String[] {"a", "b"}); + csConf.setCapacity(A, 50); + csConf.setUserLimit(A, 15); + csConf.setUserLimitFactor(A, 1.5f); + csConf.setCapacity(B, 50); // Test capacityScheduler = new CapacityScheduler(); @@ -1257,13 +1232,13 @@ public void testQueueParsingWithDefaultUserLimitValues() capacityScheduler.init(csConf); capacityScheduler.start(); Assert.assertEquals(15, - ((LeafQueue)capacityScheduler.getQueue(queueA)).getUserLimit(), DELTA); + ((LeafQueue)capacityScheduler.getQueue(A_PATH)).getUserLimit(), DELTA); Assert.assertEquals(1.5, - ((LeafQueue)capacityScheduler.getQueue(queueA)).getUserLimitFactor(), DELTA); + ((LeafQueue)capacityScheduler.getQueue(A_PATH)).getUserLimitFactor(), DELTA); Assert.assertEquals(100, - ((LeafQueue)capacityScheduler.getQueue(queueB)).getUserLimit(), DELTA); + ((LeafQueue)capacityScheduler.getQueue(B_PATH)).getUserLimit(), DELTA); Assert.assertEquals(1, - ((LeafQueue)capacityScheduler.getQueue(queueB)).getUserLimitFactor(), DELTA); + ((LeafQueue)capacityScheduler.getQueue(B_PATH)).getUserLimitFactor(), DELTA); ServiceOperations.stopQuietly(capacityScheduler); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java index aa3b5919fcc60..482f7f332b48c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueState.java @@ -53,12 +53,15 @@ public class TestQueueState { private static final String Q2 = "q2"; private static final String Q3 = "q3"; - private final static String Q1_PATH = - CapacitySchedulerConfiguration.ROOT + "." + Q1; - private final static String Q2_PATH = - Q1_PATH + "." + Q2; - private final static String Q3_PATH = - Q1_PATH + "." + Q3; + + private final static QueuePath ROOT_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT); + private final static QueuePath Q1_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT + "." + Q1); + private final static QueuePath Q2_PATH = + new QueuePath(Q1_PATH + "." + Q2); + private final static QueuePath Q3_PATH = + new QueuePath(Q1_PATH + "." + Q3); private CapacityScheduler cs; private YarnConfiguration conf; @@ -66,7 +69,7 @@ public class TestQueueState { public void testQueueState() throws IOException { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {Q1}); + csConf.setQueues(ROOT_PATH, new String[] {Q1}); csConf.setQueues(Q1_PATH, new String[] {Q2}); csConf.setCapacity(Q1_PATH, 100); @@ -120,7 +123,7 @@ public void testQueueState() throws IOException { public void testQueueStateTransit() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {Q1}); + csConf.setQueues(ROOT_PATH, new String[] {Q1}); csConf.setQueues(Q1_PATH, new String[] {Q2, Q3}); csConf.setCapacity(Q1_PATH, 100); @@ -232,7 +235,7 @@ public void testRecoverDrainingStateAfterRMRestart() throws Exception { false); newConf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); newConf.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, 1); - newConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{Q1}); + newConf.setQueues(ROOT_PATH, new String[]{Q1}); newConf.setQueues(Q1_PATH, new String[]{Q2}); newConf.setCapacity(Q1_PATH, 100); newConf.setCapacity(Q2_PATH, 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java index e893717a8dd0f..ec26a3d338b49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java @@ -53,6 +53,10 @@ public class TestQueueStateManager { Q1_PATH + "." + Q2; private final static String Q3_PATH = Q1_PATH + "." + Q3; + private final static QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private final static QueuePath Q1_QUEUE_PATH = new QueuePath(Q1_PATH); + private final static QueuePath Q2_QUEUE_PATH = new QueuePath(Q2_PATH); + private final static QueuePath Q3_QUEUE_PATH = new QueuePath(Q1_PATH); private CapacityScheduler cs; private YarnConfiguration conf; @@ -61,12 +65,12 @@ public void testQueueStateManager() throws AccessControlException, YarnException { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {Q1}); - csConf.setQueues(Q1_PATH, new String[] {Q2, Q3}); + csConf.setQueues(ROOT_QUEUE_PATH, new String[] {Q1}); + csConf.setQueues(Q1_QUEUE_PATH, new String[] {Q2, Q3}); - csConf.setCapacity(Q1_PATH, 100); - csConf.setCapacity(Q2_PATH, 50); - csConf.setCapacity(Q3_PATH, 50); + csConf.setCapacity(Q1_QUEUE_PATH, 100); + csConf.setCapacity(Q2_QUEUE_PATH, 50); + csConf.setCapacity(Q3_QUEUE_PATH, 50); conf = new YarnConfiguration(csConf); cs = new CapacityScheduler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index c6f947febc00e..e698142385490 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -168,27 +168,26 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf, final String newRoot, boolean addUserLimits) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] { newRoot }); - conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100); - conf.setAcl(CapacitySchedulerConfiguration.ROOT, - QueueACL.SUBMIT_APPLICATIONS, " "); - - final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." - + newRoot; - conf.setQueues(Q_newRoot, new String[] { A }); - conf.setCapacity(Q_newRoot, 100); - conf.setMaximumCapacity(Q_newRoot, 100); - conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " "); - - final String Q_A = Q_newRoot + "." + A; - conf.setCapacity(Q_A, 100f); - conf.setMaximumCapacity(Q_A, 100); - conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*"); + QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + QueuePath newRootPath = QueuePath.addQueueToPath(root, newRoot); + QueuePath aQueuePath = QueuePath.addQueueToPath(newRootPath, A); + + conf.setQueues(root, new String[] { newRoot }); + conf.setMaximumCapacity(root, 100); + conf.setAcl(root, QueueACL.SUBMIT_APPLICATIONS, " "); + + conf.setQueues(newRootPath, new String[] { A }); + conf.setCapacity(newRootPath, 100); + conf.setMaximumCapacity(newRootPath, 100); + conf.setAcl(newRootPath, QueueACL.SUBMIT_APPLICATIONS, " "); + + conf.setCapacity(aQueuePath, 100f); + conf.setMaximumCapacity(aQueuePath, 100); + conf.setAcl(aQueuePath, QueueACL.SUBMIT_APPLICATIONS, "*"); if (addUserLimits) { - conf.setUserLimit(Q_A, 25); - conf.setUserLimitFactor(Q_A, 0.25f); + conf.setUserLimit(aQueuePath, 25); + conf.setUserLimitFactor(aQueuePath, 0.25f); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 2cac58bd8bb22..9f928482a75b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -55,7 +55,6 @@ import java.util.Map; import java.util.Set; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_QUEUE_PATH; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -65,6 +64,21 @@ public class TestUtils { private static final Logger LOG = LoggerFactory.getLogger(TestUtils.class); + private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c"; + private static final String A1_PATH = A_PATH + ".a1"; + private static final String B1_PATH = B_PATH + ".b1"; + private static final String B2_PATH = B_PATH + ".b2"; + private static final String C1_PATH = C_PATH + ".c1"; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A = new QueuePath(A_PATH); + private static final QueuePath B = new QueuePath(B_PATH); + private static final QueuePath C = new QueuePath(C_PATH); + private static final QueuePath A1 = new QueuePath(A1_PATH); + private static final QueuePath B1 = new QueuePath(B1_PATH); + private static final QueuePath B2 = new QueuePath(B2_PATH); + private static final QueuePath C1 = new QueuePath(C1_PATH); /** * Get a mock {@link RMContext} for use in test cases. @@ -280,41 +294,35 @@ public static Configuration getConfigurationWithQueueLabels(Configuration config new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); + conf.setQueues(ROOT, new String[] {"a", "b", "c"}); + conf.setCapacityByLabel(ROOT, "x", 100); + conf.setCapacityByLabel(ROOT, "y", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); conf.setAccessibleNodeLabels(A, toSet("x")); conf.setCapacityByLabel(A, "x", 100); - - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setCapacity(B, 20); conf.setAccessibleNodeLabels(B, toSet("y")); conf.setCapacityByLabel(B, "y", 100); - - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + conf.setCapacity(C, 70); conf.setMaximumCapacity(C, 70); conf.setAccessibleNodeLabels(C, RMNodeLabelsManager.EMPTY_STRING_SET); // Define 2nd-level queues - final String A1 = A + ".a1"; conf.setQueues(A, new String[] {"a1"}); conf.setCapacity(A1, 100); conf.setMaximumCapacity(A1, 100); conf.setCapacityByLabel(A1, "x", 100); - - final String B1 = B + ".b1"; + conf.setQueues(B, new String[] {"b1"}); conf.setCapacity(B1, 100); conf.setMaximumCapacity(B1, 100); conf.setCapacityByLabel(B1, "y", 100); - conf.setMaximumApplicationMasterResourcePerQueuePercent(B1_QUEUE_PATH, 1f); + conf.setMaximumApplicationMasterResourcePerQueuePercent(B1, 1f); - final String C1 = C + ".c1"; conf.setQueues(C, new String[] {"c1"}); conf.setCapacity(C1, 100); conf.setMaximumCapacity(C1, 100); @@ -328,19 +336,17 @@ public static Configuration getComplexConfigurationWithQueueLabels( new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); - conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); + conf.setQueues(ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(ROOT, "x", 100); + conf.setCapacityByLabel(ROOT, "y", 100); + conf.setCapacityByLabel(ROOT, "z", 100); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 10); conf.setAccessibleNodeLabels(A, toSet("x", "y")); conf.setCapacityByLabel(A, "x", 100); conf.setCapacityByLabel(A, "y", 50); - - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setCapacity(B, 90); conf.setMaximumCapacity(B, 100); conf.setAccessibleNodeLabels(B, toSet("y", "z")); @@ -348,7 +354,6 @@ public static Configuration getComplexConfigurationWithQueueLabels( conf.setCapacityByLabel(B, "z", 100); // Define 2nd-level queues - final String A1 = A + ".a1"; conf.setQueues(A, new String[] {"a1"}); conf.setCapacity(A1, 100); conf.setMaximumCapacity(A1, 100); @@ -358,12 +363,10 @@ public static Configuration getComplexConfigurationWithQueueLabels( conf.setCapacityByLabel(A1, "y", 100); conf.setQueues(B, new String[] {"b1", "b2"}); - final String B1 = B + ".b1"; conf.setCapacity(B1, 50); conf.setMaximumCapacity(B1, 50); conf.setAccessibleNodeLabels(B1, RMNodeLabelsManager.EMPTY_STRING_SET); - final String B2 = B + ".b2"; conf.setCapacity(B2, 50); conf.setMaximumCapacity(B2, 50); conf.setAccessibleNodeLabels(B2, toSet("y", "z")); @@ -375,9 +378,6 @@ public static Configuration getComplexConfigurationWithQueueLabels( public static Configuration getConfigurationWithDefaultQueueLabels( Configuration config) { - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; - CapacitySchedulerConfiguration conf = (CapacitySchedulerConfiguration) getConfigurationWithQueueLabels(config); new CapacitySchedulerConfiguration(config); @@ -407,20 +407,17 @@ public static FiCaSchedulerApp getFiCaSchedulerApp(MockRM rm, new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] { "a", "b", "c" }); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 100); conf.setUserLimitFactor(A, 100); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 20); conf.setMaximumCapacity(B, 100); conf.setUserLimitFactor(B, 100); - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; conf.setCapacity(C, 70); conf.setMaximumCapacity(C, 100); conf.setUserLimitFactor(C, 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestConfigurationUpdateAssembler.java index 890996ac23e57..fa767a5182652 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestConfigurationUpdateAssembler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestConfigurationUpdateAssembler.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.junit.Before; @@ -163,7 +164,7 @@ public void testRemoveNonExistingQueue() { private CapacitySchedulerConfiguration crateInitialCSConfig() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a, b"}); + csConf.setQueues(new QueuePath(CapacitySchedulerConfiguration.ROOT), new String[] {"a, b"}); csConf.set(A_CONFIG_PATH, A_INIT_CONFIG_VALUE); csConf.set(B_CONFIG_PATH, B_INIT_CONFIG_VALUE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index aec2c21956471..6045255f6dc5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; @@ -221,12 +222,13 @@ public void testHDFSBackedProvider() throws Exception { public void testAddRemoveQueueWithSpacesInConfig() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {" a , b, c" }); + QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + QueuePath a = QueuePath.addQueueToPath(root, "a"); + QueuePath b = QueuePath.addQueueToPath(root, "b"); + QueuePath c = QueuePath.addQueueToPath(root, "c"); + + csConf.setQueues(root, new String[] {" a , b, c" }); - final String a = CapacitySchedulerConfiguration.ROOT + ".a"; - final String b = CapacitySchedulerConfiguration.ROOT + ".b"; - final String c = CapacitySchedulerConfiguration.ROOT + ".c"; csConf.setCapacity(a, 0); csConf.setCapacity(b, 50); csConf.setCapacity(c, 50); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java index 1d33165601421..79f9b884c7fe8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java @@ -54,6 +54,7 @@ public class TestQueueCapacityConfigParser { private static final String QUEUE = "root.test"; private static final QueuePath QUEUE_PATH = new QueuePath("root.test"); + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); private static final String ABSOLUTE_RESOURCE = String.format( ALL_RESOURCE_TEMPLATE, MEMORY_ABSOLUTE, VCORE_ABSOLUTE, GPU_ABSOLUTE); @@ -74,9 +75,9 @@ public class TestQueueCapacityConfigParser { @Test public void testPercentageCapacityConfig() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.setCapacity(QUEUE, PERCENTAGE_VALUE); + conf.setCapacity(QUEUE_PATH, PERCENTAGE_VALUE); - QueueCapacityVector percentageCapacityVector = capacityConfigParser.parse(conf, QUEUE, + QueueCapacityVector percentageCapacityVector = capacityConfigParser.parse(conf, QUEUE_PATH, NO_LABEL); QueueCapacityVectorEntry memory = percentageCapacityVector.getResource(MEMORY_URI); QueueCapacityVectorEntry vcore = percentageCapacityVector.getResource(VCORES_URI); @@ -87,8 +88,7 @@ public void testPercentageCapacityConfig() { Assert.assertEquals(QueueCapacityType.PERCENTAGE, vcore.getVectorResourceType()); Assert.assertEquals(PERCENTAGE_VALUE, vcore.getResourceValue(), EPSILON); - QueueCapacityVector rootCapacityVector = capacityConfigParser.parse(conf, - CapacitySchedulerConfiguration.ROOT, NO_LABEL); + QueueCapacityVector rootCapacityVector = capacityConfigParser.parse(conf, ROOT, NO_LABEL); QueueCapacityVectorEntry memoryRoot = rootCapacityVector.getResource(MEMORY_URI); QueueCapacityVectorEntry vcoreRoot = rootCapacityVector.getResource(VCORES_URI); @@ -105,7 +105,8 @@ public void testWeightCapacityConfig() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.setNonLabeledQueueWeight(QUEUE_PATH, WEIGHT_VALUE); - QueueCapacityVector weightCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + QueueCapacityVector weightCapacityVector = capacityConfigParser.parse(conf, QUEUE_PATH, + NO_LABEL); QueueCapacityVectorEntry memory = weightCapacityVector.getResource(MEMORY_URI); QueueCapacityVectorEntry vcore = weightCapacityVector.getResource(VCORES_URI); @@ -120,11 +121,12 @@ public void testWeightCapacityConfig() { @Test public void testAbsoluteCapacityVectorConfig() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE); + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE); conf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_TYPES); ResourceUtils.resetResourceTypes(conf); - QueueCapacityVector absoluteCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + QueueCapacityVector absoluteCapacityVector = capacityConfigParser.parse(conf, QUEUE_PATH, + NO_LABEL); Assert.assertEquals(QueueCapacityType.ABSOLUTE, absoluteCapacityVector.getResource(MEMORY_URI) .getVectorResourceType()); @@ -141,9 +143,10 @@ public void testAbsoluteCapacityVectorConfig() { Assert.assertEquals(GPU_ABSOLUTE, absoluteCapacityVector.getResource(GPU_URI) .getResourceValue(), EPSILON); - conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE_MEMORY_VCORE); - QueueCapacityVector withoutGpuVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + QueueCapacityVector withoutGpuVector = capacityConfigParser.parse(conf, QUEUE_PATH, + NO_LABEL); Assert.assertEquals(3, withoutGpuVector.getResourceCount()); Assert.assertEquals(0f, withoutGpuVector.getResource(GPU_URI).getResourceValue(), EPSILON); @@ -152,12 +155,12 @@ public void testAbsoluteCapacityVectorConfig() { @Test public void testMixedCapacityConfig() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY,MIXED_RESOURCE); + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY,MIXED_RESOURCE); conf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_TYPES); ResourceUtils.resetResourceTypes(conf); QueueCapacityVector mixedCapacityVector = - capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + capacityConfigParser.parse(conf, QUEUE_PATH, NO_LABEL); Assert.assertEquals(QueueCapacityType.ABSOLUTE, mixedCapacityVector.getResource(MEMORY_URI).getVectorResourceType()); @@ -175,11 +178,11 @@ public void testMixedCapacityConfig() { mixedCapacityVector.getResource(GPU_URI).getResourceValue(), EPSILON); // Test undefined capacity type default value - conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE_MEMORY_VCORE); QueueCapacityVector mixedCapacityVectorWithGpuUndefined = - capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + capacityConfigParser.parse(conf, QUEUE_PATH, NO_LABEL); Assert.assertEquals(QueueCapacityType.ABSOLUTE, mixedCapacityVectorWithGpuUndefined.getResource(MEMORY_URI).getVectorResourceType()); Assert.assertEquals(0, mixedCapacityVectorWithGpuUndefined.getResource(GPU_URI) @@ -191,47 +194,47 @@ public void testMixedCapacityConfig() { public void testInvalidCapacityConfigs() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.set(getQueuePrefix(QUEUE) + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, NONEXISTINGSUFFIX); QueueCapacityVector capacityVectorWithInvalidSuffix = - capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + capacityConfigParser.parse(conf, QUEUE_PATH, NO_LABEL); List entriesWithInvalidSuffix = Lists.newArrayList(capacityVectorWithInvalidSuffix.iterator()); Assert.assertEquals(0, entriesWithInvalidSuffix.size()); - conf.set(getQueuePrefix(QUEUE) + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, INVALID_CAPACITY_FORMAT); QueueCapacityVector invalidDelimiterCapacityVector = - capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + capacityConfigParser.parse(conf, QUEUE_PATH, NO_LABEL); List invalidDelimiterEntries = Lists.newArrayList(invalidDelimiterCapacityVector.iterator()); Assert.assertEquals(0, invalidDelimiterEntries.size()); - conf.set(getQueuePrefix(QUEUE) + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, INVALID_CAPACITY_BRACKET); QueueCapacityVector invalidCapacityVector = - capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + capacityConfigParser.parse(conf, QUEUE_PATH, NO_LABEL); List resources = Lists.newArrayList(invalidCapacityVector.iterator()); Assert.assertEquals(0, resources.size()); - conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, EMPTY_BRACKET); + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, EMPTY_BRACKET); QueueCapacityVector emptyBracketCapacityVector = - capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + capacityConfigParser.parse(conf, QUEUE_PATH, NO_LABEL); List emptyEntries = Lists.newArrayList(emptyBracketCapacityVector.iterator()); Assert.assertEquals(0, emptyEntries.size()); - conf.set(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY, ""); + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, ""); QueueCapacityVector emptyCapacity = - capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + capacityConfigParser.parse(conf, QUEUE_PATH, NO_LABEL); List emptyResources = Lists.newArrayList(emptyCapacity.iterator()); Assert.assertEquals(emptyResources.size(), 0); - conf.unset(getQueuePrefix(QUEUE) + CapacitySchedulerConfiguration.CAPACITY); + conf.unset(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY); QueueCapacityVector nonSetCapacity = - capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + capacityConfigParser.parse(conf, QUEUE_PATH, NO_LABEL); List nonSetResources = Lists.newArrayList(nonSetCapacity.iterator()); Assert.assertEquals(nonSetResources.size(), 0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java index 5d525c84639ca..837d78df8da00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; @@ -416,10 +417,14 @@ public void testSchedulerRejection() throws Exception { stopRM(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[] {"a", "b"}); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 15.0f); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".b", 85.0f); + + QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + QueuePath queueA = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".a"); + QueuePath queueB = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".b"); + + csConf.setQueues(root, new String[] {"a", "b"}); + csConf.setCapacity(queueA, 15.0f); + csConf.setCapacity(queueB, 85.0f); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java index 6599080aab59c..d74d469f5d46e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.MappingRulesDescription; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.Rule; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.Rule.FallbackResult; @@ -466,7 +467,7 @@ public void testConvertNestedDefaultWithConflictingQueues() { when(fsRule.getParentRule()).thenReturn(parent); when(fsRule.getCreateFlag()).thenReturn(true); initPlacementManagerMock(fsRule); - csConf.setQueues("root.users", new String[] {"hadoop"}); + csConf.setQueues(new QueuePath("root.users"), new String[] {"hadoop"}); convert(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 7183f7f782f56..c67ad3fefdd8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -37,6 +37,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.server.api.records.NodeStatus; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; @@ -860,8 +861,9 @@ public void testNonDefaultMinimumAllocation() throws Exception { @Test(timeout = 50000) public void testReconnectedNode() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.setQueues("default", new String[] { "default" }); - conf.setCapacity("default", 100); + QueuePath defaultQueuePath = new QueuePath("default"); + conf.setQueues(defaultQueuePath, new String[] { "default" }); + conf.setCapacity(defaultQueuePath, 100); FifoScheduler fs = new FifoScheduler(); fs.init(conf); fs.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java index 28c41c16e5c1c..9cfbb7da197d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT_QUEUE_PATH; import static org.junit.Assert.assertEquals; import java.util.*; @@ -165,12 +164,13 @@ public void testSizeBasedWeightNotAffectAppActivation() throws Exception { new CapacitySchedulerConfiguration(); // Define top-level queues - QueuePath queuePath = DEFAULT_QUEUE_PATH; + String defaultPath = CapacitySchedulerConfiguration.ROOT + ".default"; + QueuePath queuePath = new QueuePath(defaultPath); csConf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName()); - csConf.setOrderingPolicy(queuePath.getFullPath(), + csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY); - csConf.setOrderingPolicyParameter(queuePath.getFullPath(), + csConf.setOrderingPolicyParameter(queuePath, FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true"); csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 8120bb21b1eba..e8a9c08df2087 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -329,36 +330,37 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { static void setupQueueConfiguration(CapacitySchedulerConfiguration conf, boolean useDRC) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); + QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + conf.setQueues(root, new String[] {"a", "b", "c"}); - final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + final QueuePath A = QueuePath.addQueueToPath(root, "a"); conf.setCapacity(A, 10); - final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + final QueuePath B = QueuePath.addQueueToPath(root, "b"); conf.setCapacity(B, 20); - final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + final QueuePath C = QueuePath.addQueueToPath(root, "v"); conf.setCapacity(C, 70); // Define 2nd-level queues - final String A1 = A + ".a1"; - final String A2 = A + ".a2"; + final QueuePath A1 = QueuePath.addQueueToPath(A, "a1"); + final QueuePath A2 = QueuePath.addQueueToPath(A, "a2"); conf.setQueues(A, new String[] {"a1", "a2"}); conf.setCapacity(A1, 30); conf.setCapacity(A2, 70); - final String B1 = B + ".b1"; - final String B2 = B + ".b2"; - final String B3 = B + ".b3"; + final QueuePath B1 = QueuePath.addQueueToPath(B, "b1"); + final QueuePath B2 = QueuePath.addQueueToPath(B, "b2"); + final QueuePath B3 = QueuePath.addQueueToPath(B, "b3"); conf.setQueues(B, new String[] {"b1", "b2", "b3"}); conf.setCapacity(B1, 50); conf.setCapacity(B2, 30); conf.setCapacity(B3, 20); - final String C1 = C + ".c1"; - final String C2 = C + ".c2"; - final String C3 = C + ".c3"; - final String C4 = C + ".c4"; + final QueuePath C1 = QueuePath.addQueueToPath(C, "c1"); + final QueuePath C2 = QueuePath.addQueueToPath(C, "c2"); + final QueuePath C3 = QueuePath.addQueueToPath(C, "c3"); + final QueuePath C4 = QueuePath.addQueueToPath(C, "c3"); conf.setQueues(C, new String[] {"c1", "c2", "c3", "c4"}); conf.setCapacity(C1, 50); conf.setCapacity(C2, 10); @@ -366,9 +368,9 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf, conf.setCapacity(C4, 5); // Define 3rd-level queues - final String C11 = C1 + ".c11"; - final String C12 = C1 + ".c12"; - final String C13 = C1 + ".c13"; + final QueuePath C11 = QueuePath.addQueueToPath(C1, "c11"); + final QueuePath C12 = QueuePath.addQueueToPath(C1, "c12"); + final QueuePath C13 = QueuePath.addQueueToPath(C1, "c13"); conf.setQueues(C1, new String[] {"c11", "c12", "c13"}); conf.setCapacity(C11, 15); conf.setCapacity(C12, 45); @@ -405,8 +407,8 @@ public static FifoScheduler mockFifoScheduler(RMContext rmContext) static void setupFifoQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define default queue - conf.setQueues("default", new String[] {"default"}); - conf.setCapacity("default", 100); + conf.setQueues(new QueuePath("default"), new String[] {"default"}); + conf.setCapacity(new QueuePath("default"), 100); } public static void main(String[] args) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java index 7397926732e95..39e68974d1672 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; @@ -116,19 +117,19 @@ private static void setupQueueConfiguration( CapacitySchedulerConfiguration config) { // Define top-level queues - config.setQueues(CapacitySchedulerConfiguration.ROOT, + QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + QueuePath queueA = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".a"); + QueuePath defaultQueue = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".default"); + + config.setQueues(root, new String[]{"a", "default"}); - final String queueA = CapacitySchedulerConfiguration.ROOT + ".a"; config.setCapacity(queueA, 50f); config.setMaximumCapacity(queueA, 50); - final String defaultQueue = - CapacitySchedulerConfiguration.ROOT + ".default"; config.setCapacity(defaultQueue, 50f); - config.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "X", 100); - config.setMaximumCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "X", - 100); + config.setCapacityByLabel(root, "X", 100); + config.setMaximumCapacityByLabel(root, "X", 100); // set for default queue config.setCapacityByLabel(defaultQueue, "X", 100); config.setMaximumCapacityByLabel(defaultQueue, "X", 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 0c8f742fae246..62bf3d5843154 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -80,6 +80,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; @@ -146,7 +147,10 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase { "test.build.data", "/tmp")).getAbsolutePath(); private static final String FS_ALLOC_FILE = new File(TEST_DIR, "test-fs-queues.xml").getAbsolutePath(); - + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath DEFAULT = QueuePath.addQueueToPath(ROOT, "default"); + private static final QueuePath TEST = QueuePath.addQueueToPath(ROOT, "test"); + private static final QueuePath TEST_QUEUE = QueuePath.addQueueToPath(ROOT, "testqueue"); /* * Helper class to allow testing of RM web services which require * authorization Add this class as a filter in the Guice injector for the @@ -566,8 +570,8 @@ public void testSingleAppKillUnauthorized() throws Exception { // default root queue allows anyone to have admin acl CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(); - csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser"); - csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser"); + csconf.setAcl(ROOT, QueueACL.ADMINISTER_QUEUE, "someuser"); + csconf.setAcl(DEFAULT, QueueACL.ADMINISTER_QUEUE, "someuser"); rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext()); } @@ -789,9 +793,9 @@ public void testAppSubmit(String acceptMedia, String contentMedia) String[] queues = { "default", "testqueue" }; CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(); - csconf.setQueues("root", queues); - csconf.setCapacity("root.default", 50.0f); - csconf.setCapacity("root.testqueue", 50.0f); + csconf.setQueues(ROOT, queues); + csconf.setCapacity(DEFAULT, 50.0f); + csconf.setCapacity(TEST_QUEUE, 50.0f); rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext()); String appName = "test"; @@ -1097,12 +1101,12 @@ public void testUpdateAppPriority() throws Exception { CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(); String[] queues = { "default", "test" }; - csconf.setQueues("root", queues); - csconf.setCapacity("root.default", 50.0f); - csconf.setCapacity("root.test", 50.0f); - csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser"); - csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser"); - csconf.setAcl("root.test", QueueACL.ADMINISTER_QUEUE, "someuser"); + csconf.setQueues(ROOT, queues); + csconf.setCapacity(DEFAULT, 50.0f); + csconf.setCapacity(TEST, 50.0f); + csconf.setAcl(ROOT, QueueACL.ADMINISTER_QUEUE, "someuser"); + csconf.setAcl(DEFAULT, QueueACL.ADMINISTER_QUEUE, "someuser"); + csconf.setAcl(TEST, QueueACL.ADMINISTER_QUEUE, "someuser"); rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext()); rm.start(); @@ -1188,12 +1192,12 @@ public void testAppMove() throws Exception { CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(); String[] queues = { "default", "test" }; - csconf.setQueues("root", queues); - csconf.setCapacity("root.default", 50.0f); - csconf.setCapacity("root.test", 50.0f); - csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser"); - csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser"); - csconf.setAcl("root.test", QueueACL.ADMINISTER_QUEUE, "someuser"); + csconf.setQueues(ROOT, queues); + csconf.setCapacity(DEFAULT, 50.0f); + csconf.setCapacity(TEST, 50.0f); + csconf.setAcl(ROOT, QueueACL.ADMINISTER_QUEUE, "someuser"); + csconf.setAcl(DEFAULT, QueueACL.ADMINISTER_QUEUE, "someuser"); + csconf.setAcl(TEST, QueueACL.ADMINISTER_QUEUE, "someuser"); rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext()); rm.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 0c9598f25923e..082ab808395f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; @@ -82,6 +83,29 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase { + private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; + private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; + private static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c"; + private static final String A1_PATH = A_PATH + ".a1"; + private static final String A2_PATH = A_PATH + ".a2"; + private static final String B1_PATH = B_PATH + ".b1"; + private static final String B2_PATH = B_PATH + ".b2"; + private static final String B3_PATH = B_PATH + ".b3"; + private static final String A1A_PATH = A1_PATH + ".a1a"; + private static final String A1B_PATH = A1_PATH + ".a1b"; + private static final String A1C_PATH = A1_PATH + ".a1c"; + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A = new QueuePath(A_PATH); + private static final QueuePath B = new QueuePath(B_PATH); + private static final QueuePath C = new QueuePath(C_PATH); + private static final QueuePath A1 = new QueuePath(A1_PATH); + private static final QueuePath A2 = new QueuePath(A2_PATH); + private static final QueuePath B1 = new QueuePath(B1_PATH); + private static final QueuePath B2 = new QueuePath(B2_PATH); + private static final QueuePath B3 = new QueuePath(B3_PATH); + private static final QueuePath A1A = new QueuePath(A1A_PATH); + private static final QueuePath A1B = new QueuePath(A1B_PATH); + private static final QueuePath A1C = new QueuePath(A1C_PATH); private MockRM rm; public static class WebServletModule extends ServletModule { @@ -119,54 +143,43 @@ public static void setupQueueConfiguration( CapacitySchedulerConfiguration config) { // Define top-level queues - config.setQueues(CapacitySchedulerConfiguration.ROOT, + config.setQueues(ROOT, new String[] {"a", "b", "c"}); - final String a = CapacitySchedulerConfiguration.ROOT + ".a"; - config.setCapacity(a, 10.5f); - config.setMaximumCapacity(a, 50); - config.setInt(QueuePrefixes.getQueuePrefix(a) + MAX_PARALLEL_APPLICATIONS, 42); + config.setCapacity(A, 10.5f); + config.setMaximumCapacity(A, 50); + config.setInt(QueuePrefixes.getQueuePrefix(A) + MAX_PARALLEL_APPLICATIONS, 42); - final String b = CapacitySchedulerConfiguration.ROOT + ".b"; - config.setCapacity(b, 89.5f); + config.setCapacity(B, 89.5f); - final String c = CapacitySchedulerConfiguration.ROOT + ".c"; - config.setCapacity(c, "[memory=1024]"); + config.setCapacity(C, "[memory=1024]"); // Define 2nd-level queues - final String a1 = a + ".a1"; - final String a2 = a + ".a2"; - config.setQueues(a, new String[] {"a1", "a2"}); - config.setCapacity(a1, 30); - config.setMaximumCapacity(a1, 50); - config.setMaximumLifetimePerQueue(a2, 100); - config.setDefaultLifetimePerQueue(a2, 50); - - config.setUserLimitFactor(a1, 100.0f); - config.setCapacity(a2, 70); - config.setUserLimitFactor(a2, 100.0f); - - final String b1 = b + ".b1"; - final String b2 = b + ".b2"; - final String b3 = b + ".b3"; - config.setQueues(b, new String[] {"b1", "b2", "b3"}); - config.setCapacity(b1, 60); - config.setUserLimitFactor(b1, 100.0f); - config.setCapacity(b2, 39.5f); - config.setUserLimitFactor(b2, 100.0f); - config.setCapacity(b3, 0.5f); - config.setUserLimitFactor(b3, 100.0f); - - config.setQueues(a1, new String[] {"a1a", "a1b", "a1c"}); - final String a1A = a1 + ".a1a"; - config.setCapacity(a1A, 65); - final String a1B = a1 + ".a1b"; - config.setCapacity(a1B, 15); - final String a1C = a1 + ".a1c"; - config.setCapacity(a1C, 20); - - config.setAutoCreateChildQueueEnabled(a1C, true); - config.setInt(PREFIX + a1C + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + config.setQueues(A, new String[] {"a1", "a2"}); + config.setCapacity(A1, 30); + config.setMaximumCapacity(A1, 50); + config.setMaximumLifetimePerQueue(A2, 100); + config.setDefaultLifetimePerQueue(A2, 50); + + config.setUserLimitFactor(A1, 100.0f); + config.setCapacity(A2, 70); + config.setUserLimitFactor(A2, 100.0f); + + config.setQueues(B, new String[] {"b1", "b2", "b3"}); + config.setCapacity(B1, 60); + config.setUserLimitFactor(B1, 100.0f); + config.setCapacity(B2, 39.5f); + config.setUserLimitFactor(B2, 100.0f); + config.setCapacity(B3, 0.5f); + config.setUserLimitFactor(B3, 100.0f); + + config.setQueues(A1, new String[] {"a1a", "a1b", "a1c"}); + config.setCapacity(A1A, 65); + config.setCapacity(A1B, 15); + config.setCapacity(A1C, 20); + + config.setAutoCreateChildQueueEnabled(A1C, true); + config.setInt(PREFIX + A1C + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + DOT + CAPACITY, 50); } @@ -239,8 +252,8 @@ public void testNodeLabelDefaultAPI() throws Exception { CapacitySchedulerConfiguration config = ((CapacityScheduler)rm.getResourceScheduler()).getConfiguration(); - config.setDefaultNodeLabelExpression("root", "ROOT-INHERITED"); - config.setDefaultNodeLabelExpression("root.a", "root-a-default-label"); + config.setDefaultNodeLabelExpression(ROOT, "ROOT-INHERITED"); + config.setDefaultNodeLabelExpression(A, "root-a-default-label"); rm.getResourceScheduler().reinitialize(config, rm.getRMContext()); //Start RM so that it accepts app submissions diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 5055b3efcfa8d..2d512abe87ae9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; +import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.junit.Test; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; @@ -146,8 +147,9 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() throws Exception { Configuration config = CSConfigGenerator .createWeightConfigWithAutoQueueCreationEnabled(); + QueuePath autoPArent1 = new QueuePath("root.autoParent1"); config.setInt(QueuePrefixes - .getQueuePrefix("root.autoParent1") + + .getQueuePrefix(autoPArent1) + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + "maximum-applications", 300); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 9579ce25f34f8..a98ed2ffe3083 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -68,7 +68,6 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_QUEUE_PATH; import static org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils.toJson; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -92,6 +91,10 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase { public static final QueuePath ROOT_A = new QueuePath("root", "a"); public static final QueuePath ROOT_A_A1 = QueuePath.createFromQueues("root", "a", "a1"); public static final QueuePath ROOT_A_A2 = QueuePath.createFromQueues("root", "a", "a2"); + public static final QueuePath ROOT_B = new QueuePath("root", "b"); + public static final QueuePath ROOT_C = new QueuePath("root", "c"); + public static final QueuePath ROOT_C_C1 = QueuePath.createFromQueues("root", "c", "c1"); + public static final QueuePath ROOT_D = new QueuePath("root", "d"); private static MockRM rm; private static String userName; private static CapacitySchedulerConfiguration csConf; @@ -148,31 +151,23 @@ public void setUp() throws Exception { private static void setupQueueConfiguration( CapacitySchedulerConfiguration config) { - config.setQueues(CapacitySchedulerConfiguration.ROOT, - new String[]{"a", "b", "c", "mappedqueue"}); + config.setQueues(ROOT, new String[]{"a", "b", "c", "mappedqueue"}); - final String a = CapacitySchedulerConfiguration.ROOT + ".a"; - config.setCapacity(a, 25f); - config.setMaximumCapacity(a, 50f); + config.setCapacity(ROOT_A, 25f); + config.setMaximumCapacity(ROOT_A, 50f); - final String a1 = a + ".a1"; - final String a2 = a + ".a2"; - config.setQueues(a, new String[]{"a1", "a2"}); - config.setCapacity(a1, 100f); - config.setCapacity(a2, 0f); + config.setQueues(ROOT_A, new String[]{"a1", "a2"}); + config.setCapacity(ROOT_A_A1, 100f); + config.setCapacity(ROOT_A_A2, 0f); - final String b = CapacitySchedulerConfiguration.ROOT + ".b"; - config.setCapacity(b, 75f); + config.setCapacity(ROOT_B, 75f); - final String c = CapacitySchedulerConfiguration.ROOT + ".c"; - config.setCapacity(c, 0f); + config.setCapacity(ROOT_C, 0f); - final String c1 = c + ".c1"; - config.setQueues(c, new String[] {"c1"}); - config.setCapacity(c1, 0f); + config.setQueues(ROOT_C, new String[] {"c1"}); + config.setCapacity(ROOT_C_C1, 0f); - final String d = CapacitySchedulerConfiguration.ROOT + ".d"; - config.setCapacity(d, 0f); + config.setCapacity(ROOT_D, 0f); config.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, "g:hadoop:mappedqueue"); } @@ -210,14 +205,14 @@ private CapacitySchedulerConfiguration getSchedulerConf() public void testGetSchedulerConf() throws Exception { CapacitySchedulerConfiguration orgConf = getSchedulerConf(); assertNotNull(orgConf); - assertEquals(4, orgConf.getQueues("root").length); + assertEquals(4, orgConf.getQueues(ROOT).length); } @Test public void testFormatSchedulerConf() throws Exception { CapacitySchedulerConfiguration newConf = getSchedulerConf(); assertNotNull(newConf); - assertEquals(4, newConf.getQueues("root").length); + assertEquals(4, newConf.getQueues(ROOT).length); SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map nearEmptyCapacity = new HashMap<>(); @@ -243,7 +238,7 @@ public void testFormatSchedulerConf() throws Exception { .put(ClientResponse.class); newConf = getSchedulerConf(); assertNotNull(newConf); - assertEquals(5, newConf.getQueues("root").length); + assertEquals(5, newConf.getQueues(ROOT).length); // Format the scheduler config and validate root.formattest is not present response = r.path("ws").path("v1").path("cluster") @@ -252,7 +247,7 @@ public void testFormatSchedulerConf() throws Exception { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); newConf = getSchedulerConf(); - assertEquals(4, newConf.getQueues("root").length); + assertEquals(4, newConf.getQueues(ROOT).length); } private long getConfigVersion() throws Exception { @@ -278,7 +273,7 @@ public void testSchedulerConfigVersion() throws Exception { public void testAddNestedQueue() throws Exception { CapacitySchedulerConfiguration orgConf = getSchedulerConf(); assertNotNull(orgConf); - assertEquals(4, orgConf.getQueues("root").length); + assertEquals(4, orgConf.getQueues(ROOT).length); WebResource r = resource(); @@ -313,8 +308,8 @@ public void testAddNestedQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(5, newCSConf.getQueues("root").length); - assertEquals(2, newCSConf.getQueues("root.d").length); + assertEquals(5, newCSConf.getQueues(ROOT).length); + assertEquals(2, newCSConf.getQueues(ROOT_D).length); assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d.d1")), 0.01f); assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d.d2")), @@ -322,7 +317,7 @@ public void testAddNestedQueue() throws Exception { CapacitySchedulerConfiguration newConf = getSchedulerConf(); assertNotNull(newConf); - assertEquals(5, newConf.getQueues("root").length); + assertEquals(5, newConf.getQueues(ROOT).length); } @Test @@ -352,7 +347,7 @@ public void testAddWithUpdate() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(5, newCSConf.getQueues("root").length); + assertEquals(5, newCSConf.getQueues(ROOT).length); assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d")), 0.01f); assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f); } @@ -382,7 +377,7 @@ public void testUnsetParentQueueOrderingPolicy() throws Exception { + "root.b" + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY; assertEquals("fair", newCSConf.get(bOrderingPolicy)); - stopQueue("root.b"); + stopQueue(ROOT_B); // Add root.b.b1 which makes root.b a Parent Queue SchedConfUpdateInfo updateInfo2 = new SchedConfUpdateInfo(); @@ -433,7 +428,7 @@ public void testUnsetLeafQueueOrderingPolicy() throws Exception { + "root.c" + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY; assertEquals("priority-utilization", newCSConf.get(cOrderingPolicy)); - stopQueue("root.c.c1"); + stopQueue(ROOT_C_C1); // Remove root.c.c1 which makes root.c a Leaf Queue SchedConfUpdateInfo updateInfo2 = new SchedConfUpdateInfo(); @@ -462,7 +457,7 @@ public void testRemoveQueue() throws Exception { ClientResponse response; - stopQueue("root.a.a2"); + stopQueue(ROOT_A_A2); // Remove root.a.a2 SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.a.a2"); @@ -478,9 +473,9 @@ public void testRemoveQueue() throws Exception { CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals("Failed to remove the queue", - 1, newCSConf.getQueues("root.a").length); + 1, newCSConf.getQueues(ROOT_A).length); assertEquals("Failed to remove the right queue", - "a1", newCSConf.getQueues("root.a")[0]); + "a1", newCSConf.getQueues(ROOT_A)[0]); } @Test @@ -509,8 +504,8 @@ public void testStopWithRemoveQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(1, newCSConf.getQueues("root.a").length); - assertEquals("a1", newCSConf.getQueues("root.a")[0]); + assertEquals(1, newCSConf.getQueues(ROOT_A).length); + assertEquals("a1", newCSConf.getQueues(ROOT_A)[0]); } @Test @@ -548,7 +543,7 @@ public void testRemoveQueueWhichHasQueueMapping() throws Exception { // Validate queue 'mappedqueue' exists after above failure CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(4, newCSConf.getQueues("root").length); + assertEquals(4, newCSConf.getQueues(ROOT).length); assertNotNull("CapacityScheduler Configuration is corrupt", cs.getQueue("mappedqueue")); } @@ -582,8 +577,8 @@ public void testStopWithConvertLeafToParentQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(1, newCSConf.getQueues("root.b").length); - assertEquals("b1", newCSConf.getQueues("root.b")[0]); + assertEquals(1, newCSConf.getQueues(ROOT_B).length); + assertEquals("b1", newCSConf.getQueues(ROOT_B)[0]); } @Test @@ -592,7 +587,7 @@ public void testRemoveParentQueue() throws Exception { ClientResponse response; - stopQueue("root.c", "root.c.c1"); + stopQueue(ROOT_C, ROOT_C_C1); // Remove root.c (parent queue) SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.c"); @@ -607,8 +602,8 @@ public void testRemoveParentQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(3, newCSConf.getQueues("root").length); - assertNull(newCSConf.getQueues("root.c")); + assertEquals(3, newCSConf.getQueues(ROOT).length); + assertNull(newCSConf.getQueues(ROOT_C)); } @Test @@ -617,7 +612,7 @@ public void testRemoveParentQueueWithCapacity() throws Exception { ClientResponse response; - stopQueue("root.a", "root.a.a1", "root.a.a2"); + stopQueue(ROOT_A, ROOT_A_A1, ROOT_A_A2); // Remove root.a (parent queue) with capacity 25 SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.a"); @@ -638,7 +633,7 @@ public void testRemoveParentQueueWithCapacity() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(3, newCSConf.getQueues("root").length); + assertEquals(3, newCSConf.getQueues(ROOT_B).length); assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f); } @@ -649,7 +644,7 @@ public void testRemoveMultipleQueues() throws Exception { ClientResponse response; - stopQueue("root.b", "root.c", "root.c.c1"); + stopQueue(ROOT_B, ROOT_C, ROOT_C_C1); // Remove root.b and root.c SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); updateInfo.getRemoveQueueInfo().add("root.b"); @@ -670,10 +665,10 @@ public void testRemoveMultipleQueues() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(2, newCSConf.getQueues("root").length); + assertEquals(2, newCSConf.getQueues(ROOT).length); } - private void stopQueue(String... queuePaths) throws Exception { + private void stopQueue(QueuePath... queuePaths) throws Exception { WebResource r = resource(); ClientResponse response; @@ -683,8 +678,8 @@ private void stopQueue(String... queuePaths) throws Exception { Map stoppedParam = new HashMap<>(); stoppedParam.put(CapacitySchedulerConfiguration.STATE, QueueState.STOPPED.toString()); - for (String queue : queuePaths) { - QueueConfigInfo stoppedInfo = new QueueConfigInfo(queue, stoppedParam); + for (QueuePath queue : queuePaths) { + QueueConfigInfo stoppedInfo = new QueueConfigInfo(queue.getFullPath(), stoppedParam); updateInfo.getUpdateQueueInfo().add(stoppedInfo); } response = @@ -697,7 +692,7 @@ private void stopQueue(String... queuePaths) throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - for (String queue : queuePaths) { + for (QueuePath queue : queuePaths) { assertEquals(QueueState.STOPPED, newCSConf.getState(queue)); } } @@ -720,7 +715,7 @@ public void testUpdateQueue() throws Exception { assertEquals(CapacitySchedulerConfiguration .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, cs.getConfiguration() - .getMaximumApplicationMasterResourcePerQueuePercent(A_QUEUE_PATH), + .getMaximumApplicationMasterResourcePerQueuePercent(ROOT_A), 0.001f); response = r.path("ws").path("v1").path("cluster") @@ -733,7 +728,7 @@ public void testUpdateQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = cs.getConfiguration(); assertEquals(0.2f, newCSConf - .getMaximumApplicationMasterResourcePerQueuePercent(A_QUEUE_PATH), 0.001f); + .getMaximumApplicationMasterResourcePerQueuePercent(ROOT_A), 0.001f); // Remove config. Config value should be reverted to default. updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX, @@ -752,7 +747,7 @@ public void testUpdateQueue() throws Exception { newCSConf = cs.getConfiguration(); assertEquals(CapacitySchedulerConfiguration .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, newCSConf - .getMaximumApplicationMasterResourcePerQueuePercent(A_QUEUE_PATH), + .getMaximumApplicationMasterResourcePerQueuePercent(ROOT_A), 0.001f); } @@ -888,9 +883,9 @@ public void testNodeLabelRemovalResidualConfigsAreCleared() throws Exception { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); assertEquals(Sets.newHashSet("*"), - cs.getConfiguration().getAccessibleNodeLabels(ROOT.getFullPath())); + cs.getConfiguration().getAccessibleNodeLabels(ROOT)); assertEquals(Sets.newHashSet(LABEL_1), - cs.getConfiguration().getAccessibleNodeLabels(ROOT_A.getFullPath())); + cs.getConfiguration().getAccessibleNodeLabels(ROOT_A)); // 4. Set partition capacities to queues as below updateInfo = new SchedConfUpdateInfo(); @@ -987,8 +982,8 @@ public void testNodeLabelRemovalResidualConfigsAreCleared() throws Exception { .put(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); assertEquals(Sets.newHashSet("*"), - cs.getConfiguration().getAccessibleNodeLabels(ROOT.getFullPath())); - assertNull(cs.getConfiguration().getAccessibleNodeLabels(ROOT_A.getFullPath())); + cs.getConfiguration().getAccessibleNodeLabels(ROOT)); + assertNull(cs.getConfiguration().getAccessibleNodeLabels(ROOT_A)); //6. Remove node label 'label1' MultivaluedMapImpl params = new MultivaluedMapImpl(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 6af918070419d..bfce233a0994a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; @@ -99,6 +100,7 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase { private static final String QUEUE_A = "Qa"; private static final String LABEL_LY = "Ly"; private static final String LABEL_LX = "Lx"; + private static final QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); private static final ImmutableSet CLUSTER_LABELS = ImmutableSet.of(LABEL_LX, LABEL_LY, DEFAULT_PARTITION); private static final String DOT = "."; @@ -142,60 +144,63 @@ private static void setupQueueConfiguration( CapacitySchedulerConfiguration config) { // Define top-level queues - config.setQueues(CapacitySchedulerConfiguration.ROOT, + config.setQueues(ROOT_QUEUE_PATH, new String[] { QUEUE_A, QUEUE_B, QUEUE_C }); String interMediateQueueC = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_C; - config.setQueues(interMediateQueueC, + QueuePath interMediateQueueCPath = new QueuePath(interMediateQueueC); + config.setQueues(interMediateQueueCPath, new String[] { LEAF_QUEUE_C1, LEAF_QUEUE_C2 }); - config.setCapacityByLabel( - CapacitySchedulerConfiguration.ROOT, LABEL_LX, 100); - config.setCapacityByLabel( - CapacitySchedulerConfiguration.ROOT, LABEL_LY, 100); + config.setCapacityByLabel(ROOT_QUEUE_PATH, LABEL_LX, 100); + config.setCapacityByLabel(ROOT_QUEUE_PATH, LABEL_LY, 100); String leafQueueA = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_A; - config.setCapacity(leafQueueA, 30); - config.setMaximumCapacity(leafQueueA, 50); + QueuePath leafQueueAPath = new QueuePath(leafQueueA); + config.setCapacity(leafQueueAPath, 30); + config.setMaximumCapacity(leafQueueAPath, 50); String leafQueueB = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_B; - config.setCapacity(leafQueueB, 30); - config.setMaximumCapacity(leafQueueB, 50); + QueuePath leafQueueBPath = new QueuePath(leafQueueA); + config.setCapacity(leafQueueBPath, 30); + config.setMaximumCapacity(leafQueueBPath, 50); - config.setCapacity(interMediateQueueC, 40); - config.setMaximumCapacity(interMediateQueueC, 50); + config.setCapacity(interMediateQueueCPath, 40); + config.setMaximumCapacity(interMediateQueueCPath, 50); String leafQueueC1 = interMediateQueueC + "." + LEAF_QUEUE_C1; - config.setCapacity(leafQueueC1, 50); - config.setMaximumCapacity(leafQueueC1, 60); + QueuePath leafQueueC1Path = new QueuePath(leafQueueA); + config.setCapacity(leafQueueC1Path, 50); + config.setMaximumCapacity(leafQueueC1Path, 60); String leafQueueC2 = interMediateQueueC + "." + LEAF_QUEUE_C2; - config.setCapacity(leafQueueC2, 50); - config.setMaximumCapacity(leafQueueC2, 70); + QueuePath leafQueueC2Path = new QueuePath(leafQueueA); + config.setCapacity(leafQueueC2Path, 50); + config.setMaximumCapacity(leafQueueC2Path, 70); // Define label specific configuration config.setAccessibleNodeLabels( - leafQueueA, ImmutableSet.of(DEFAULT_PARTITION)); - config.setAccessibleNodeLabels(leafQueueB, ImmutableSet.of(LABEL_LX)); - config.setAccessibleNodeLabels(interMediateQueueC, + leafQueueAPath, ImmutableSet.of(DEFAULT_PARTITION)); + config.setAccessibleNodeLabels(leafQueueBPath, ImmutableSet.of(LABEL_LX)); + config.setAccessibleNodeLabels(interMediateQueueCPath, ImmutableSet.of(LABEL_LX, LABEL_LY)); - config.setAccessibleNodeLabels(leafQueueC1, + config.setAccessibleNodeLabels(leafQueueC1Path, ImmutableSet.of(LABEL_LX, LABEL_LY)); - config.setAccessibleNodeLabels(leafQueueC2, + config.setAccessibleNodeLabels(leafQueueC2Path, ImmutableSet.of(LABEL_LX, LABEL_LY)); - config.setDefaultNodeLabelExpression(leafQueueB, LABEL_LX); - config.setDefaultNodeLabelExpression(leafQueueC1, LABEL_LX); - config.setDefaultNodeLabelExpression(leafQueueC2, LABEL_LY); - - config.setCapacityByLabel(leafQueueB, LABEL_LX, 30); - config.setCapacityByLabel(interMediateQueueC, LABEL_LX, 70); - config.setCapacityByLabel(leafQueueC1, LABEL_LX, 40); - config.setCapacityByLabel(leafQueueC2, LABEL_LX, 60); - - config.setCapacityByLabel(interMediateQueueC, LABEL_LY, 100); - config.setCapacityByLabel(leafQueueC1, LABEL_LY, 50); - config.setCapacityByLabel(leafQueueC2, LABEL_LY, 50); - config.setMaximumCapacityByLabel(leafQueueC1, LABEL_LY, 75); - config.setMaximumCapacityByLabel(leafQueueC2, LABEL_LY, 75); + config.setDefaultNodeLabelExpression(leafQueueBPath, LABEL_LX); + config.setDefaultNodeLabelExpression(leafQueueC1Path, LABEL_LX); + config.setDefaultNodeLabelExpression(leafQueueC2Path, LABEL_LY); + + config.setCapacityByLabel(leafQueueBPath, LABEL_LX, 30); + config.setCapacityByLabel(interMediateQueueCPath, LABEL_LX, 70); + config.setCapacityByLabel(leafQueueC1Path, LABEL_LX, 40); + config.setCapacityByLabel(leafQueueC2Path, LABEL_LX, 60); + + config.setCapacityByLabel(interMediateQueueCPath, LABEL_LY, 100); + config.setCapacityByLabel(leafQueueC1Path, LABEL_LY, 50); + config.setCapacityByLabel(leafQueueC2Path, LABEL_LY, 50); + config.setMaximumCapacityByLabel(leafQueueC1Path, LABEL_LY, 75); + config.setMaximumCapacityByLabel(leafQueueC2Path, LABEL_LY, 75); } @Before diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java index a0481283f0f05..9989c4a377e73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; @@ -173,9 +174,9 @@ public void configureScheduler() { CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(conf); String[] queues = { "default", "dedicated" }; - csconf.setQueues("root", queues); - csconf.setCapacity("root.default", 50.0f); - csconf.setCapacity("root.dedicated", 50.0f); + csconf.setQueues(new QueuePath("root"), queues); + csconf.setCapacity(new QueuePath("root.default"), 50.0f); + csconf.setCapacity(new QueuePath("root.dedicated"), 50.0f); csconf.setReservable("root.dedicated", true); conf = csconf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java index ac75abeab0aab..017233cbf463f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivitiesWithMultiNodesEnabled.java @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.ResourceUsageMultiNodeLookupPolicy; import org.apache.hadoop.yarn.util.resource.Resources; @@ -59,7 +60,6 @@ import java.util.List; import java.util.function.Predicate; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_ACT_ALLOCATIONS; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_ACT_ALLOCATION_STATE; import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.FN_ACT_COUNT; @@ -138,14 +138,18 @@ protected void configureServlets() { private static void setupQueueConfiguration( CapacitySchedulerConfiguration config) { // Define top-level queues - config.setQueues(CapacitySchedulerConfiguration.ROOT, + final QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + final QueuePath a = new QueuePath(CapacitySchedulerConfiguration.ROOT, "a"); + final QueuePath b = new QueuePath(CapacitySchedulerConfiguration.ROOT, "b"); + + config.setQueues(root, new String[] {"a", "b"}); - config.setCapacity(A, 10.5f); - config.setMaximumCapacity(A, 50); + config.setCapacity(a, 10.5f); + config.setMaximumCapacity(a, 50); - config.setCapacity(B, 89.5f); - config.setMaximumApplicationMasterResourcePerQueuePercent(B_QUEUE_PATH, 100); + config.setCapacity(b, 89.5f); + config.setMaximumApplicationMasterResourcePerQueuePercent(b, 100); } @Before diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java index 423e0e5a38cbe..6f55449265fe4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; @@ -81,6 +82,9 @@ public abstract class BaseRouterWebServicesTest { public static final String QUEUE_DEDICATED = "dedicated"; public static final String QUEUE_DEDICATED_FULL = CapacitySchedulerConfiguration.ROOT + CapacitySchedulerConfiguration.DOT + QUEUE_DEDICATED; + public static final QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); + public static final QueuePath DEFAULT_QUEUE_PATH = new QueuePath(QUEUE_DEFAULT_FULL); + public static final QueuePath DEDICATED_QUEUE_PATH = new QueuePath(QUEUE_DEDICATED_FULL); private RouterWebServices routerWebService; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java index e2ac5fbf260cd..350ed98eb9a40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java @@ -152,10 +152,7 @@ import org.slf4j.LoggerFactory; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEFAULT; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEFAULT_FULL; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED_FULL; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -1086,11 +1083,11 @@ private MockRM setupResourceManager() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); // Define default queue - conf.setCapacity(QUEUE_DEFAULT_FULL, 20); + conf.setCapacity(DEFAULT_QUEUE_PATH, 20); // Define dedicated queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT_QUEUE_PATH, new String[] {QUEUE_DEFAULT, QUEUE_DEDICATED}); - conf.setCapacity(QUEUE_DEDICATED_FULL, 80); + conf.setCapacity(DEDICATED_QUEUE_PATH, 80); conf.setReservable(QUEUE_DEDICATED_FULL, true); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java index 31fd756b664eb..61fb43d861e89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java @@ -30,13 +30,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED_FULL; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEFAULT_FULL; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEFAULT; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.*; /** * Extends the FederationInterceptorREST and overrides methods to provide a @@ -82,11 +80,11 @@ protected void setupResourceManager() throws IOException { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); // Define default queue - conf.setCapacity(QUEUE_DEFAULT_FULL, 20); + conf.setCapacity(DEFAULT_QUEUE_PATH, 20); // Define dedicated queues String[] queues = new String[]{QUEUE_DEFAULT, QUEUE_DEDICATED}; - conf.setQueues(CapacitySchedulerConfiguration.ROOT, queues); - conf.setCapacity(QUEUE_DEDICATED_FULL, 80); + conf.setQueues(ROOT_QUEUE_PATH, queues); + conf.setCapacity(DEFAULT_QUEUE_PATH, 80); conf.setReservable(QUEUE_DEDICATED_FULL, true); conf.setClass(YarnConfiguration.RM_SCHEDULER, From 8cd7931fb99fcf78c7bfa1be602d2463c160e1ac Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Mon, 13 Feb 2023 10:13:47 +0100 Subject: [PATCH 04/21] YARN-11041. Refactored template config settings Change-Id: I532c5244a411ac230664058c2e92cad1002978fc --- .../capacity/AutoCreatedQueueTemplate.java | 63 +++++++------------ .../scheduler/capacity/QueuePath.java | 62 ++++++++++++++++-- .../scheduler/capacity/QueuePrefixes.java | 2 +- .../nodelabels/TestRMNodeLabelsManager.java | 2 +- .../rmapp/TestApplicationLifetimeMonitor.java | 4 +- .../TestAutoCreatedQueueTemplate.java | 4 +- .../scheduler/capacity/TestParentQueue.java | 26 ++++---- .../scheduler/capacity/TestReservations.java | 4 +- .../TestMutableCSConfigurationProvider.java | 6 +- .../resourcemanager/webapp/TestRMWebApp.java | 30 ++++----- .../TestRMWebServicesAppsModification.java | 6 +- 11 files changed, 120 insertions(+), 89 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index c8099190081cf..eaa4716a2dd56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -41,7 +41,7 @@ public class AutoCreatedQueueTemplate { public static final String AUTO_QUEUE_PARENT_TEMPLATE_PREFIX = AUTO_QUEUE_CREATION_V2_PREFIX + "parent-template."; - private static final String WILDCARD_QUEUE = "*"; + public static final String WILDCARD_QUEUE = "*"; private final Map templateProperties = new HashMap<>(); private final Map leafOnlyProperties = new HashMap<>(); @@ -104,7 +104,7 @@ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, QueuePath childQueuePath, boolean isLeaf) { - if (childQueuePath.equals(ROOT)) { + if (childQueuePath.getFullPath().equals(ROOT)) { return; } @@ -150,47 +150,26 @@ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, */ private void setTemplateConfigEntries(CapacitySchedulerConfiguration configuration, QueuePath queuePath) { - ConfigurationProperties configurationProperties = - configuration.getConfigurationProperties(); - - List queuePathParts = Lists.newArrayList(queuePath.iterator()); - - if (queuePathParts.size() <= 1 && !queuePath.isRoot()) { - // This is an invalid queue path - return; - } - int queuePathMaxIndex = queuePathParts.size() - 1; - - // start with the most explicit format (without wildcard) - int wildcardLevel = 0; - // root can not be wildcarded - int supportedWildcardLevel = Math.min(queuePathMaxIndex, - configuration.getMaximumAutoCreatedQueueDepth(queuePath)); - // Allow root to have template properties - if (queuePath.isRoot()) { - supportedWildcardLevel = 0; - } - - // Collect all template entries - while (wildcardLevel <= supportedWildcardLevel) { - String queuePathString = String.join(".", queuePathParts); - QueuePath templateQueuePath = new QueuePath(queuePathString); - // Get all configuration entries with - // yarn.scheduler.capacity. prefix - Map queueProps = configurationProperties - .getPropertiesWithPrefix(getQueuePrefix(templateQueuePath)); - - // Store template, parent-template and leaf-template properties - for (Map.Entry entry : queueProps.entrySet()) { - storeConfiguredTemplates(entry.getKey(), entry.getValue()); + if (!queuePath.isInvalid()) { + ConfigurationProperties configurationProperties = + configuration.getConfigurationProperties(); + + int maxAutoCreatedQueueDepth = configuration + .getMaximumAutoCreatedQueueDepth(queuePath); + List wildcardedQueuePaths = + queuePath.getWildcardedQueuePaths(maxAutoCreatedQueueDepth); + + for (QueuePath templateQueuePath: wildcardedQueuePaths) { + // Get all configuration entries with + // yarn.scheduler.capacity. prefix + Map queueProps = configurationProperties + .getPropertiesWithPrefix(getQueuePrefix(templateQueuePath)); + + // Store template, parent-template and leaf-template properties + for (Map.Entry entry : queueProps.entrySet()) { + storeConfiguredTemplates(entry.getKey(), entry.getValue()); + } } - - // Replace a queue part with a wildcard based on the wildcard level - // eg. root.a -> root.* - int queuePartToWildcard = queuePathMaxIndex - wildcardLevel; - queuePathParts.set(queuePartToWildcard, WILDCARD_QUEUE); - - ++wildcardLevel; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index 808f9765fc01f..81dd6cc957d72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -18,11 +18,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import com.google.common.collect.Iterators; + +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; +import java.util.List; import java.util.NoSuchElementException; import java.util.Objects; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.WILDCARD_QUEUE; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; @@ -61,13 +66,13 @@ public QueuePath(String fullPath) { } /** - * Static method to create Queue path from parent QueuePath object and a leaf queue name. - * @param parent Parent QueuePath object for the queue - * @param leaf Name of the leaf queue + * Creates a new {@code QueuePath} with the current object as a parent, and appends the + * child to its queue path + * @param childName Name of the child which will be added to the queue path * @return QueuePath object */ - public static QueuePath addQueueToPath(QueuePath parent, String leaf) { - return new QueuePath(parent.getFullPath(), leaf); + public QueuePath createChild(String childName) { + return new QueuePath(getFullPath(), childName); } /** @@ -115,6 +120,14 @@ public boolean hasEmptyPart() { return false; } + /** + * Simple helper method to determine if the queue path is invalid or not. + * @return true if the queue path is invalid. + */ + public boolean isInvalid() { + return getFullPath().split(QUEUE_REGEX_DELIMITER).length <= 1 && !isRoot(); + } + /** * Getter for the parent part of the path. * @return Parent path of the queue, null if there is no parent. @@ -217,6 +230,45 @@ public String next() { }; } + /** + * Returns the list of wildcarded queue paths based on the autoCreatedQueueDepth config value. + * An example template precedence hierarchy for root.a ParentQueue from highest to lowest: + * yarn.scheduler.capacity.root.a.auto-queue-creation-v2.template.capacity + * yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity + * @param maxAutoCreatedQueueDepth the maximum depth of auto-created queues stored in the + * configuration + * @return list of wildcarded QueuePath objects + */ + public List getWildcardedQueuePaths(int maxAutoCreatedQueueDepth) { + List wildcardedPaths = new ArrayList<>(); + // Start with the most explicit format (without wildcard) + wildcardedPaths.add(this); + + String[] pathParts = getFullPath().split(QUEUE_REGEX_DELIMITER); + int supportedWildcardLevel = getSupportedWildcardLevel(maxAutoCreatedQueueDepth); + + // Collect all template entries + for (int wildcardLevel = supportedWildcardLevel; wildcardLevel > 0; wildcardLevel--) { + pathParts[wildcardLevel] = WILDCARD_QUEUE; + String wildcardedPath = String.join(DOT, pathParts); + wildcardedPaths.add(new QueuePath(wildcardedPath)); + } + + return wildcardedPaths; + } + + /** + * Returns the supported wildcard level for this queuepath + * @param maxAutoCreatedQueueDepth the maximum depth of auto-created queues stored in the + * configuration + * @return int value of the supported wildcard level + */ + private int getSupportedWildcardLevel(int maxAutoCreatedQueueDepth) { + int queuePathMaxIndex = Iterators.size(iterator()) - 1; + // Allow root to have template properties + return isRoot() ? 0 : Math.min(queuePathMaxIndex, maxAutoCreatedQueueDepth); + } + @Override public String toString() { return getFullPath(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java index 9d15ca67e8288..e5fdabe790919 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java @@ -36,6 +36,6 @@ public static String getAutoCreatedQueueTemplateConfPrefix(QueuePath queuePath) } public static QueuePath getAutoCreatedQueueObjectTemplateConfPrefix(QueuePath queuePath) { - return new QueuePath(queuePath.getFullPath(), AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX); + return queuePath.createChild(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index a4444b4d479b3..bfd3b0ca6c10d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -672,7 +672,7 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { conf.setQueues(root, new String[] { "a" }); conf.setCapacityByLabel(root, "x", 100); - final QueuePath A = QueuePath.addQueueToPath(root, "a"); + final QueuePath A = root.createNewLeaf("a"); conf.setCapacity(A, 100); conf.setAccessibleNodeLabels(A, ImmutableSet.of("x")); conf.setCapacityByLabel(A, "x", 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java index 0f1919b93baa2..569919c01c4fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java @@ -78,8 +78,8 @@ public class TestApplicationLifetimeMonitor { private final long maxLifetime = 30L; private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); private static final String CQ1 = "child1"; - private static final QueuePath CQ1_QUEUE_PATH = QueuePath.addQueueToPath(ROOT, CQ1); - private static final QueuePath DEFAULT_QUEUE_PATH = QueuePath.addQueueToPath(ROOT, "default"); + private static final QueuePath CQ1_QUEUE_PATH = ROOT.createNewLeaf(CQ1); + private static final QueuePath DEFAULT_QUEUE_PATH = ROOT.createNewLeaf("default"); private YarnConfiguration conf; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index 8454650ecf73f..4e81cd5c20259 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -123,7 +123,7 @@ public void testWildcardAfterRoot() { public void testTemplatePrecedence() { conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w"); conf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "4w"); - conf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "2w"); + conf.set(getTemplateKey(TEST_QUEUE_TWO_LEVEL_WILDCARDS, "capacity"), "2w"); AutoCreatedQueueTemplate template = new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB); @@ -149,7 +149,7 @@ public void testRootTemplate() { conf.set(getTemplateKey(ROOT, "capacity"), "2w"); AutoCreatedQueueTemplate template = - new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A); + new AutoCreatedQueueTemplate(conf, ROOT); template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); Assert.assertEquals("root property is not set", 2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 0967e9763f9bf..0d415dc9690b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -101,19 +101,19 @@ public class TestParentQueue { private static final String B2 = "b2"; private static final String B3 = "b3"; private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); - private static final QueuePath Q_A = QueuePath.addQueueToPath(ROOT, A); - private static final QueuePath Q_B = QueuePath.addQueueToPath(ROOT, B); - private static final QueuePath Q_C = QueuePath.addQueueToPath(ROOT, C); - private static final QueuePath Q_D = QueuePath.addQueueToPath(ROOT, D); - private static final QueuePath Q_A1 = QueuePath.addQueueToPath(Q_A, A1); - private static final QueuePath Q_A2 = QueuePath.addQueueToPath(Q_A, A2); - private static final QueuePath Q_B1 = QueuePath.addQueueToPath(Q_B, B1); - private static final QueuePath Q_B2 = QueuePath.addQueueToPath(Q_B, B2); - private static final QueuePath Q_B3 = QueuePath.addQueueToPath(Q_B, B3); - private static final QueuePath Q_C1 = QueuePath.addQueueToPath(Q_C, C1); - private static final QueuePath Q_C11 = QueuePath.addQueueToPath(Q_C1, C11); - private static final QueuePath Q_C111 = QueuePath.addQueueToPath(Q_C11, C111); - private static final QueuePath Q_C1111 = QueuePath.addQueueToPath(Q_C111, C1111); + private static final QueuePath Q_A = ROOT.createNewLeaf(A); + private static final QueuePath Q_B = ROOT.createNewLeaf(B); + private static final QueuePath Q_C = ROOT.createNewLeaf(C); + private static final QueuePath Q_D = ROOT.createNewLeaf(D); + private static final QueuePath Q_A1 = Q_A.createNewLeaf(A1); + private static final QueuePath Q_A2 = Q_A.createNewLeaf(A2); + private static final QueuePath Q_B1 = Q_B.createNewLeaf(B1); + private static final QueuePath Q_B2 = Q_B.createNewLeaf(B2); + private static final QueuePath Q_B3 = Q_B.createNewLeaf(B3); + private static final QueuePath Q_C1 = Q_C.createNewLeaf(C1); + private static final QueuePath Q_C11 = Q_C1.createNewLeaf(C11); + private static final QueuePath Q_C111 = Q_C11.createNewLeaf(C111); + private static final QueuePath Q_C1111 = Q_C111.createNewLeaf(C1111); @Before public void setUp() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index e698142385490..9037be4e495a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -169,8 +169,8 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf, // Define top-level queues QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); - QueuePath newRootPath = QueuePath.addQueueToPath(root, newRoot); - QueuePath aQueuePath = QueuePath.addQueueToPath(newRootPath, A); + QueuePath newRootPath = root.createNewLeaf(newRoot); + QueuePath aQueuePath = newRootPath.createNewLeaf(A); conf.setQueues(root, new String[] { newRoot }); conf.setMaximumCapacity(root, 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 6045255f6dc5a..89b7dc4b2ff84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -223,9 +223,9 @@ public void testAddRemoveQueueWithSpacesInConfig() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); - QueuePath a = QueuePath.addQueueToPath(root, "a"); - QueuePath b = QueuePath.addQueueToPath(root, "b"); - QueuePath c = QueuePath.addQueueToPath(root, "c"); + QueuePath a = root.createNewLeaf("a"); + QueuePath b = root.createNewLeaf("b"); + QueuePath c = root.createNewLeaf("c"); csConf.setQueues(root, new String[] {" a , b, c" }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index e8a9c08df2087..18c86ae8f79d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -333,34 +333,34 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf, QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); conf.setQueues(root, new String[] {"a", "b", "c"}); - final QueuePath A = QueuePath.addQueueToPath(root, "a"); + final QueuePath A = root.createNewLeaf("a"); conf.setCapacity(A, 10); - final QueuePath B = QueuePath.addQueueToPath(root, "b"); + final QueuePath B = root.createNewLeaf("b"); conf.setCapacity(B, 20); - final QueuePath C = QueuePath.addQueueToPath(root, "v"); + final QueuePath C = root.createNewLeaf("v"); conf.setCapacity(C, 70); // Define 2nd-level queues - final QueuePath A1 = QueuePath.addQueueToPath(A, "a1"); - final QueuePath A2 = QueuePath.addQueueToPath(A, "a2"); + final QueuePath A1 = A.createNewLeaf("a1"); + final QueuePath A2 = A.createNewLeaf("a2"); conf.setQueues(A, new String[] {"a1", "a2"}); conf.setCapacity(A1, 30); conf.setCapacity(A2, 70); - final QueuePath B1 = QueuePath.addQueueToPath(B, "b1"); - final QueuePath B2 = QueuePath.addQueueToPath(B, "b2"); - final QueuePath B3 = QueuePath.addQueueToPath(B, "b3"); + final QueuePath B1 = B.createNewLeaf("b1"); + final QueuePath B2 = B.createNewLeaf("b2"); + final QueuePath B3 = B.createNewLeaf("b3"); conf.setQueues(B, new String[] {"b1", "b2", "b3"}); conf.setCapacity(B1, 50); conf.setCapacity(B2, 30); conf.setCapacity(B3, 20); - final QueuePath C1 = QueuePath.addQueueToPath(C, "c1"); - final QueuePath C2 = QueuePath.addQueueToPath(C, "c2"); - final QueuePath C3 = QueuePath.addQueueToPath(C, "c3"); - final QueuePath C4 = QueuePath.addQueueToPath(C, "c3"); + final QueuePath C1 = C.createNewLeaf("c1"); + final QueuePath C2 = C.createNewLeaf("c2"); + final QueuePath C3 = C.createNewLeaf("c3"); + final QueuePath C4 = C.createNewLeaf("c3"); conf.setQueues(C, new String[] {"c1", "c2", "c3", "c4"}); conf.setCapacity(C1, 50); conf.setCapacity(C2, 10); @@ -368,9 +368,9 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf, conf.setCapacity(C4, 5); // Define 3rd-level queues - final QueuePath C11 = QueuePath.addQueueToPath(C1, "c11"); - final QueuePath C12 = QueuePath.addQueueToPath(C1, "c12"); - final QueuePath C13 = QueuePath.addQueueToPath(C1, "c13"); + final QueuePath C11 = C1.createNewLeaf("c11"); + final QueuePath C12 = C1.createNewLeaf("c12"); + final QueuePath C13 = C1.createNewLeaf("c13"); conf.setQueues(C1, new String[] {"c11", "c12", "c13"}); conf.setCapacity(C11, 15); conf.setCapacity(C12, 45); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 62bf3d5843154..a4c5176995709 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -148,9 +148,9 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase { private static final String FS_ALLOC_FILE = new File(TEST_DIR, "test-fs-queues.xml").getAbsolutePath(); private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); - private static final QueuePath DEFAULT = QueuePath.addQueueToPath(ROOT, "default"); - private static final QueuePath TEST = QueuePath.addQueueToPath(ROOT, "test"); - private static final QueuePath TEST_QUEUE = QueuePath.addQueueToPath(ROOT, "testqueue"); + private static final QueuePath DEFAULT = ROOT.createNewLeaf("default"); + private static final QueuePath TEST = ROOT.createNewLeaf("test"); + private static final QueuePath TEST_QUEUE = ROOT.createNewLeaf("testqueue"); /* * Helper class to allow testing of RM web services which require * authorization Add this class as a filter in the Guice injector for the From 0e5d499cab11fb115112f31b95da3bc0094fdf50 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Mon, 13 Feb 2023 10:17:39 +0100 Subject: [PATCH 05/21] YARN-11041. Refactored template config settings Change-Id: Id5fc099521dcd0411e3db324fba3d5f2a0493c02 --- .../server/resourcemanager/scheduler/capacity/QueuePath.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index 81dd6cc957d72..981c905596b94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -250,15 +250,14 @@ public List getWildcardedQueuePaths(int maxAutoCreatedQueueDepth) { // Collect all template entries for (int wildcardLevel = supportedWildcardLevel; wildcardLevel > 0; wildcardLevel--) { pathParts[wildcardLevel] = WILDCARD_QUEUE; - String wildcardedPath = String.join(DOT, pathParts); - wildcardedPaths.add(new QueuePath(wildcardedPath)); + wildcardedPaths.add(createFromQueues(pathParts)); } return wildcardedPaths; } /** - * Returns the supported wildcard level for this queuepath + * Returns the supported wildcard level for this queue path * @param maxAutoCreatedQueueDepth the maximum depth of auto-created queues stored in the * configuration * @return int value of the supported wildcard level From ca58278269f400ab5e076fac9afe9a46b2d3d14d Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Mon, 13 Feb 2023 13:51:59 +0100 Subject: [PATCH 06/21] YARN-11041. Refactored getQueues method in CapacitySchedulerConfiguration Change-Id: I2275d216e15fa5ba9ee9752de749cbfc7ffac980 --- .../CapacitySchedulerConfiguration.java | 5 +-- .../CapacitySchedulerQueueManager.java | 4 +- .../scheduler/capacity/QueuePath.java | 23 +++++++---- .../conf/ConfigurationUpdateAssembler.java | 4 +- .../converter/QueuePlacementConverter.java | 4 +- ...estRMWebServicesConfigurationMutation.java | 38 +++++++++---------- 6 files changed, 40 insertions(+), 38 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 8f7db5c7979a2..1a72c0241292b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1092,7 +1092,7 @@ public List getPriorityAcls(QueuePath queue, return priorityACLConfig.getPriorityAcl(clusterMaxPriority, aclString); } - public String[] getQueues(QueuePath queue) { + public List getQueues(QueuePath queue) { LOG.debug("CSConf - getQueues called for: queuePrefix={}", getQueuePrefix(queue)); String[] queues = getStrings(getQueuePrefix(queue) + QUEUES); @@ -1101,14 +1101,13 @@ public String[] getQueues(QueuePath queue) { for (String s : queues) { trimmedQueueNames.add(s.trim()); } - queues = trimmedQueueNames.toArray(new String[0]); } LOG.debug("CSConf - getQueues: queuePrefix={}, queues={}", getQueuePrefix(queue), ((queues == null) ? "" : StringUtils.arrayToString(queues))); - return queues; + return trimmedQueueNames; } public void setQueues(QueuePath queue, String[] subQueues) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index d344a064a7b27..4ce65f9683205 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -238,9 +238,7 @@ static CSQueue parseQueue( CSQueue queue; QueuePath fullQueuePath = (parent == null) ? new QueuePath(queueName) : (QueuePath.createFromQueues(parent.getQueuePath(), queueName)); - String[] staticChildQueueNames = conf.getQueues(fullQueuePath); - List childQueueNames = staticChildQueueNames != null ? - Arrays.asList(staticChildQueueNames) : Collections.emptyList(); + List childQueueNames = conf.getQueues(fullQueuePath); CSQueue oldQueue = oldQueues.get(fullQueuePath.getFullPath()); boolean isReservableQueue = conf.isReservable(fullQueuePath.getFullPath()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index 981c905596b94..8d0c910cb44cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import com.google.common.collect.Iterators; - import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -125,7 +123,7 @@ public boolean hasEmptyPart() { * @return true if the queue path is invalid. */ public boolean isInvalid() { - return getFullPath().split(QUEUE_REGEX_DELIMITER).length <= 1 && !isRoot(); + return getPathComponents().length <= 1 && !isRoot(); } /** @@ -193,7 +191,7 @@ public QueuePath createNewLeaf(String childQueue) { */ @Override public Iterator iterator() { - return Arrays.asList(getFullPath().split(QUEUE_REGEX_DELIMITER)).iterator(); + return Arrays.asList(getPathComponents()).iterator(); } /** @@ -244,13 +242,14 @@ public List getWildcardedQueuePaths(int maxAutoCreatedQueueDepth) { // Start with the most explicit format (without wildcard) wildcardedPaths.add(this); - String[] pathParts = getFullPath().split(QUEUE_REGEX_DELIMITER); + String[] pathComponents = getPathComponents(); int supportedWildcardLevel = getSupportedWildcardLevel(maxAutoCreatedQueueDepth); // Collect all template entries for (int wildcardLevel = supportedWildcardLevel; wildcardLevel > 0; wildcardLevel--) { - pathParts[wildcardLevel] = WILDCARD_QUEUE; - wildcardedPaths.add(createFromQueues(pathParts)); + pathComponents[wildcardLevel] = WILDCARD_QUEUE; + QueuePath wildcardedPath = createFromQueues(pathComponents); + wildcardedPaths.add(wildcardedPath); } return wildcardedPaths; @@ -263,11 +262,19 @@ public List getWildcardedQueuePaths(int maxAutoCreatedQueueDepth) { * @return int value of the supported wildcard level */ private int getSupportedWildcardLevel(int maxAutoCreatedQueueDepth) { - int queuePathMaxIndex = Iterators.size(iterator()) - 1; + int queuePathMaxIndex = getPathComponents().length - 1; // Allow root to have template properties return isRoot() ? 0 : Math.min(queuePathMaxIndex, maxAutoCreatedQueueDepth); } + /** + * Returns queue path components. + * @return String array containing the queue names. + */ + public String[] getPathComponents() { + return getFullPath().split(QUEUE_REGEX_DELIMITER); + } + @Override public String toString() { return getFullPath(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java index 99c1c22d0a081..1f286ed1edccf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java @@ -122,9 +122,7 @@ private static void addQueue( } String parentPath = queuePath.substring(0, queuePath.lastIndexOf('.')); QueuePath parentQueuePath = new QueuePath(parentPath); - String[] siblings = proposedConf.getQueues(parentQueuePath); - List siblingQueues = siblings == null ? new ArrayList<>() : - new ArrayList<>(Arrays.asList(siblings)); + List siblingQueues = proposedConf.getQueues(parentQueuePath); siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1)); proposedConf.setQueues(parentQueuePath, siblingQueues.toArray(new String[0])); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java index 3d6797e648185..10777a61bd58c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java @@ -234,11 +234,11 @@ private Rule createNestedRule(Policy policy, private void checkStaticDynamicConflict(QueuePath parentPath, CapacitySchedulerConfiguration csConf, FSConfigToCSConfigRuleHandler ruleHandler) { - String[] childQueues = csConf.getQueues(parentPath); + List childQueues = csConf.getQueues(parentPath); // User must be warned: static + dynamic queues are under the // same parent - if (childQueues != null && childQueues.length > 0) { + if (childQueues != null && childQueues.size() > 0) { ruleHandler.handleChildStaticDynamicConflict(parentPath.getFullPath()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index a98ed2ffe3083..9cec010dde62a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -205,14 +205,14 @@ private CapacitySchedulerConfiguration getSchedulerConf() public void testGetSchedulerConf() throws Exception { CapacitySchedulerConfiguration orgConf = getSchedulerConf(); assertNotNull(orgConf); - assertEquals(4, orgConf.getQueues(ROOT).length); + assertEquals(4, orgConf.getQueues(ROOT).size()); } @Test public void testFormatSchedulerConf() throws Exception { CapacitySchedulerConfiguration newConf = getSchedulerConf(); assertNotNull(newConf); - assertEquals(4, newConf.getQueues(ROOT).length); + assertEquals(4, newConf.getQueues(ROOT).size()); SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo(); Map nearEmptyCapacity = new HashMap<>(); @@ -238,7 +238,7 @@ public void testFormatSchedulerConf() throws Exception { .put(ClientResponse.class); newConf = getSchedulerConf(); assertNotNull(newConf); - assertEquals(5, newConf.getQueues(ROOT).length); + assertEquals(5, newConf.getQueues(ROOT).size()); // Format the scheduler config and validate root.formattest is not present response = r.path("ws").path("v1").path("cluster") @@ -247,7 +247,7 @@ public void testFormatSchedulerConf() throws Exception { .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(Status.OK.getStatusCode(), response.getStatus()); newConf = getSchedulerConf(); - assertEquals(4, newConf.getQueues(ROOT).length); + assertEquals(4, newConf.getQueues(ROOT).size()); } private long getConfigVersion() throws Exception { @@ -273,7 +273,7 @@ public void testSchedulerConfigVersion() throws Exception { public void testAddNestedQueue() throws Exception { CapacitySchedulerConfiguration orgConf = getSchedulerConf(); assertNotNull(orgConf); - assertEquals(4, orgConf.getQueues(ROOT).length); + assertEquals(4, orgConf.getQueues(ROOT).size()); WebResource r = resource(); @@ -308,8 +308,8 @@ public void testAddNestedQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(5, newCSConf.getQueues(ROOT).length); - assertEquals(2, newCSConf.getQueues(ROOT_D).length); + assertEquals(5, newCSConf.getQueues(ROOT).size()); + assertEquals(2, newCSConf.getQueues(ROOT_D).size()); assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d.d1")), 0.01f); assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d.d2")), @@ -317,7 +317,7 @@ public void testAddNestedQueue() throws Exception { CapacitySchedulerConfiguration newConf = getSchedulerConf(); assertNotNull(newConf); - assertEquals(5, newConf.getQueues(ROOT).length); + assertEquals(5, newConf.getQueues(ROOT).size()); } @Test @@ -347,7 +347,7 @@ public void testAddWithUpdate() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(5, newCSConf.getQueues(ROOT).length); + assertEquals(5, newCSConf.getQueues(ROOT).size()); assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.d")), 0.01f); assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f); } @@ -473,9 +473,9 @@ public void testRemoveQueue() throws Exception { CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals("Failed to remove the queue", - 1, newCSConf.getQueues(ROOT_A).length); + 1, newCSConf.getQueues(ROOT_A).size()); assertEquals("Failed to remove the right queue", - "a1", newCSConf.getQueues(ROOT_A)[0]); + "a1", newCSConf.getQueues(ROOT_A).get(0)); } @Test @@ -504,8 +504,8 @@ public void testStopWithRemoveQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(1, newCSConf.getQueues(ROOT_A).length); - assertEquals("a1", newCSConf.getQueues(ROOT_A)[0]); + assertEquals(1, newCSConf.getQueues(ROOT_A).size()); + assertEquals("a1", newCSConf.getQueues(ROOT_A).get(0)); } @Test @@ -543,7 +543,7 @@ public void testRemoveQueueWhichHasQueueMapping() throws Exception { // Validate queue 'mappedqueue' exists after above failure CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(4, newCSConf.getQueues(ROOT).length); + assertEquals(4, newCSConf.getQueues(ROOT).size()); assertNotNull("CapacityScheduler Configuration is corrupt", cs.getQueue("mappedqueue")); } @@ -577,8 +577,8 @@ public void testStopWithConvertLeafToParentQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(1, newCSConf.getQueues(ROOT_B).length); - assertEquals("b1", newCSConf.getQueues(ROOT_B)[0]); + assertEquals(1, newCSConf.getQueues(ROOT_B).size()); + assertEquals("b1", newCSConf.getQueues(ROOT_B).get(0)); } @Test @@ -602,7 +602,7 @@ public void testRemoveParentQueue() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(3, newCSConf.getQueues(ROOT).length); + assertEquals(3, newCSConf.getQueues(ROOT).size()); assertNull(newCSConf.getQueues(ROOT_C)); } @@ -633,7 +633,7 @@ public void testRemoveParentQueueWithCapacity() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(3, newCSConf.getQueues(ROOT_B).length); + assertEquals(3, newCSConf.getQueues(ROOT_B).size()); assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f); } @@ -665,7 +665,7 @@ public void testRemoveMultipleQueues() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(2, newCSConf.getQueues(ROOT).length); + assertEquals(2, newCSConf.getQueues(ROOT).size()); } private void stopQueue(QueuePath... queuePaths) throws Exception { From 7978640313d65afae020e302f9fc0d6194c5252d Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Mon, 13 Feb 2023 15:21:37 +0100 Subject: [PATCH 07/21] YARN-11041. Fixed code formatting Change-Id: I404aa88232a5ab5c8b30c0cc663da2a2700d9e4d --- .../scheduler/capacity/AbstractCSQueue.java | 2 +- .../capacity/AutoCreatedQueueTemplate.java | 2 +- .../CapacitySchedulerConfiguration.java | 22 +++++++++---------- .../capacity/TestCapacityScheduler.java | 20 ++++++++++++++++- .../TestCapacitySchedulerDynamicBehavior.java | 17 +++++++++++++- 5 files changed, 48 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 9e9ffaf208349..a33a57f7b7b84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -387,7 +387,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws // Store preemption settings this.preemptionSettings = new CSQueuePreemptionSettings(this, configuration); this.priority = configuration.getQueuePriority( - getQueuePathObject()); + getQueuePathObject()); // Update multi-node sorting algorithm for scheduling as configured. setMultiNodeSortingPolicyName( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index eaa4716a2dd56..003bcacd08508 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -104,7 +104,7 @@ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, QueuePath childQueuePath, boolean isLeaf) { - if (childQueuePath.getFullPath().equals(ROOT)) { + if (childQueuePath.isRoot()) { return; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 1a72c0241292b..a2f7b7f90c995 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -22,7 +22,6 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueCapacityConfigParser; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.MappingRuleCreator; import org.slf4j.Logger; @@ -78,7 +77,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.*; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getAutoCreatedQueueObjectTemplateConfPrefix; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getNodeLabelPrefix; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix; public class CapacitySchedulerConfiguration extends ReservationSchedulerConfiguration { @@ -583,7 +584,7 @@ public float getNonLabeledQueueCapacity(QueuePath queue) { } public void setCapacity(QueuePath queue, float capacity) { - if (queue.getFullPath().equals("root")) { + if (queue.isRoot()) { throw new IllegalArgumentException( "Cannot set capacity, root queue has a fixed capacity of 100.0f"); } @@ -595,7 +596,7 @@ public void setCapacity(QueuePath queue, float capacity) { @VisibleForTesting public void setCapacity(QueuePath queue, String absoluteResourceCapacity) { - if (queue.getFullPath().equals("root")) { + if (queue.isRoot()) { throw new IllegalArgumentException( "Cannot set capacity, root queue has a fixed capacity"); } @@ -669,7 +670,7 @@ public float getUserLimit(QueuePath queue) { // if possible. @SuppressWarnings("unchecked") public OrderingPolicy getAppOrderingPolicy( - QueuePath queue) { + QueuePath queue) { String policyType = get(getQueuePrefix(queue) + ORDERING_POLICY, DEFAULT_APP_ORDERING_POLICY); @@ -933,7 +934,7 @@ private static String getAclKey(AccessType acl) { * @return A mapping of the queue ACLs. */ public Map getACLsForLegacyAutoCreatedLeafQueue( - QueuePath parentQueuePath) { + QueuePath parentQueuePath) { final String prefix = getQueuePrefix(getAutoCreatedQueueObjectTemplateConfPrefix( parentQueuePath)); @@ -2294,7 +2295,7 @@ public boolean isAutoQueueCreationV2Enabled(QueuePath queuePath) { */ @Private public boolean getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded( - QueuePath queuePath) { + QueuePath queuePath) { boolean shouldFailAutoQueueCreationOnExceedingGuaranteedCapacity = getBoolean(getQueuePrefix(queuePath) + FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY, @@ -2458,14 +2459,14 @@ public String getAutoCreatedQueueManagementPolicy(QueuePath queue) { * Get The policy class configured to manage capacities for auto created leaf * queues under the specified parent * - * @param queue The parent queue's name + * @param queue The parent queue's path * @return The policy class configured to manage capacities for auto created * leaf queues under the specified parent queue */ @Private protected AutoCreatedQueueManagementPolicy getAutoCreatedQueueManagementPolicyClass( - QueuePath queue) { + QueuePath queue) { String queueManagementPolicyClassName = getAutoCreatedQueueManagementPolicy(queue); @@ -2668,8 +2669,7 @@ public Map parseConfiguredResourceVector( QueuePath queuePath, Set labels) { Map queueResourceVectors = new HashMap<>(); for (String label : labels) { - queueResourceVectors.put(label, queueCapacityConfigParser.parse(this, - queuePath, label)); + queueResourceVectors.put(label, queueCapacityConfigParser.parse(this, queuePath, label)); } return queueResourceVectors; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 2eeb10ebe1fa2..6b0c229c78c02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -23,7 +23,25 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMaxAllocVcores; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMinAllocMb; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.setMinAllocVcores; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.findQueue; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupAdditionalQueues; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupBlockedQueueConfiguration; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupOtherBlockedQueueConfiguration; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A3; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_PATH; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_PATH; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkApplicationResourceUsage; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index 9260e81bf7083..afd49aae58163 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -18,7 +18,22 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.getDefaultCapacities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; From 8d102fc1fbc80ff9f6c46640a7dae507bf2b0d46 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Fri, 24 Feb 2023 16:13:39 +0100 Subject: [PATCH 08/21] YARN-11041. Fixed conflicts Change-Id: Ic9ce9d8658bbd0fdf610e36b3f0291ac61cd0300 --- .../scheduler/capacity/AbstractCSQueue.java | 2 +- .../CapacitySchedulerConfiguration.java | 10 +++--- ...citySchedulerQueueCalculationTestBase.java | 33 ++++++++++--------- .../capacity/QueueAssertionBuilder.java | 18 +++++----- .../TestMixedQueueResourceCalculation.java | 6 ++-- .../TestUniformQueueResourceCalculation.java | 12 +++---- .../conf/TestQueueCapacityConfigParser.java | 3 +- 7 files changed, 42 insertions(+), 42 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 3de68443da248..bc93032ab8c11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -381,7 +381,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws .parseConfiguredResourceVector(queuePath, this.queueNodeLabelsSettings.getConfiguredNodeLabels()); this.configuredMaxCapacityVectors = configuration - .parseConfiguredMaximumCapacityVector(queuePath.getFullPath(), + .parseConfiguredMaximumCapacityVector(queuePath, this.queueNodeLabelsSettings.getConfiguredNodeLabels(), QueueCapacityVector.newInstance()); // Update metrics diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 2957c938abaa9..6cd63c4d3d1c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -810,12 +810,12 @@ public Set getAccessibleNodeLabels(QueuePath queue) { return Collections.unmodifiableSet(set); } - public void setCapacityVector(String queuePath, String label, String capacityVector) { + public void setCapacityVector(QueuePath queuePath, String label, String capacityVector) { String capacityPropertyName = getNodeLabelPrefix(queuePath, label) + CAPACITY; set(capacityPropertyName, capacityVector); } - public void setMaximumCapacityVector(String queuePath, String label, String capacityVector) { + public void setMaximumCapacityVector(QueuePath queuePath, String label, String capacityVector) { String capacityPropertyName = getNodeLabelPrefix(queuePath, label) + MAXIMUM_CAPACITY; set(capacityPropertyName, capacityVector); } @@ -2687,7 +2687,7 @@ public Map parseConfiguredResourceVector( QueuePath queuePath, Set labels) { Map queueResourceVectors = new HashMap<>(); for (String label : labels) { - String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix( + String propertyName = QueuePrefixes.getNodeLabelPrefix( queuePath, label) + CapacitySchedulerConfiguration.CAPACITY; String capacityString = get(propertyName); queueResourceVectors.put(label, queueCapacityConfigParser.parse(capacityString, queuePath)); @@ -2697,10 +2697,10 @@ public Map parseConfiguredResourceVector( } public Map parseConfiguredMaximumCapacityVector( - String queuePath, Set labels, QueueCapacityVector defaultVector) { + QueuePath queuePath, Set labels, QueueCapacityVector defaultVector) { Map queueResourceVectors = new HashMap<>(); for (String label : labels) { - String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix( + String propertyName = QueuePrefixes.getNodeLabelPrefix( queuePath, label) + CapacitySchedulerConfiguration.MAXIMUM_CAPACITY; String capacityString = get(propertyName); QueueCapacityVector capacityVector = queueCapacityConfigParser.parse(capacityString, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueCalculationTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueCalculationTestBase.java index f62945c7a5a11..932ca753c9d30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueCalculationTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueCalculationTestBase.java @@ -33,14 +33,15 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerAutoCreatedQueueBase.GB; public class CapacitySchedulerQueueCalculationTestBase { - protected static final String A = "root.a"; - protected static final String A1 = "root.a.a1"; - protected static final String A11 = "root.a.a1.a11"; - protected static final String A12 = "root.a.a1.a12"; - protected static final String A2 = "root.a.a2"; - protected static final String B = "root.b"; - protected static final String B1 = "root.b.b1"; - protected static final String C = "root.c"; + protected static final QueuePath ROOT = new QueuePath("root"); + protected static final QueuePath A = new QueuePath("root.a"); + protected static final QueuePath A1 = new QueuePath("root.a.a1"); + protected static final QueuePath A11 = new QueuePath("root.a.a1.a11"); + protected static final QueuePath A12 = new QueuePath("root.a.a1.a12"); + protected static final QueuePath A2 = new QueuePath("root.a.a2"); + protected static final QueuePath B = new QueuePath("root.b"); + protected static final QueuePath B1 = new QueuePath("root.b.b1"); + protected static final QueuePath C = new QueuePath("root.c"); private static final String CAPACITY_VECTOR_TEMPLATE = "[memory=%s, vcores=%s]"; @@ -57,14 +58,14 @@ public void setUp() throws Exception { csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); - csConf.setQueues("root", new String[]{"a", "b"}); - csConf.setCapacity("root.a", 50f); - csConf.setCapacity("root.b", 50f); - csConf.setQueues("root.a", new String[]{"a1", "a2"}); - csConf.setCapacity("root.a.a1", 100f); - csConf.setQueues("root.a.a1", new String[]{"a11", "a12"}); - csConf.setCapacity("root.a.a1.a11", 50f); - csConf.setCapacity("root.a.a1.a12", 50f); + csConf.setQueues(ROOT, new String[]{"a", "b"}); + csConf.setCapacity(A, 50f); + csConf.setCapacity(B, 50f); + csConf.setQueues(A, new String[]{"a1", "a2"}); + csConf.setCapacity(A1, 100f); + csConf.setQueues(A1, new String[]{"a11", "a12"}); + csConf.setCapacity(A11, 50f); + csConf.setCapacity(A12, 50f); mgr = new NullRMNodeLabelsManager(); mgr.init(csConf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAssertionBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAssertionBuilder.java index 1c066719dd090..329d8b1bedbd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAssertionBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAssertionBuilder.java @@ -64,15 +64,15 @@ class QueueAssertionBuilder { } public class QueueAssertion { - private final String queuePath; + private final QueuePath queuePath; private final List assertions = new ArrayList<>(); - QueueAssertion(String queuePath) { + QueueAssertion(QueuePath queuePath) { this.queuePath = queuePath; } - public QueueAssertion withQueue(String queuePath) { + public QueueAssertion withQueue(QueuePath queuePath) { return QueueAssertionBuilder.this.withQueue(queuePath); } @@ -139,7 +139,7 @@ public void setLabel(String label) { public void withResourceSupplier( BiFunction assertion, String messageInfo) { - CSQueue queue = cs.getQueue(queuePath); + CSQueue queue = cs.getQueue(queuePath.getFullPath()); if (queue == null) { Assert.fail("Queue " + queuePath + " is not found"); } @@ -150,7 +150,7 @@ public void withResourceSupplier( public void withCapacitySupplier( BiFunction assertion, String messageInfo) { - CSQueue queue = cs.getQueue(queuePath); + CSQueue queue = cs.getQueue(queuePath.getFullPath()); if (queue == null) { Assert.fail("Queue " + queuePath + " is not found"); } @@ -161,7 +161,7 @@ public void withCapacitySupplier( } - private final Map assertions = new LinkedHashMap<>(); + private final Map assertions = new LinkedHashMap<>(); public QueueAssertionBuilder build() { return this; @@ -172,7 +172,7 @@ public QueueAssertionBuilder build() { * @param queuePath path of the queue * @return queue assertion group */ - public QueueAssertion withQueue(String queuePath) { + public QueueAssertion withQueue(QueuePath queuePath) { assertions.putIfAbsent(queuePath, new QueueAssertion(queuePath)); return assertions.get(queuePath); } @@ -181,7 +181,7 @@ public QueueAssertion withQueue(String queuePath) { * Executes assertions created for all queues. */ public void finishAssertion() { - for (Map.Entry assertionEntry : assertions.entrySet()) { + for (Map.Entry assertionEntry : assertions.entrySet()) { for (QueueAssertion.ValueAssertion assertion : assertionEntry.getValue().assertions) { if (assertion.resourceSupplier != null) { String errorMessage = String.format(RESOURCE_ASSERTION_ERROR_MESSAGE, @@ -204,7 +204,7 @@ public void finishAssertion() { * Returns all queues that have defined assertions. * @return queue paths */ - public Set getQueues() { + public Set getQueues() { return assertions.keySet(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestMixedQueueResourceCalculation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestMixedQueueResourceCalculation.java index e5b7cc964e3ab..7ff5141668e39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestMixedQueueResourceCalculation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestMixedQueueResourceCalculation.java @@ -522,15 +522,15 @@ private void setupQueueHierarchyWithWarnings() throws IOException { } private void setQueues() { - csConf.setQueues("root", new String[]{"a", "b", "c"}); + csConf.setQueues(ROOT, new String[]{"a", "b", "c"}); csConf.setQueues(A, new String[]{"a1", "a2"}); csConf.setQueues(B, new String[]{"b1"}); } private Optional getSpecificWarning( Collection warnings, QueueUpdateWarningType warningTypeToSelect, - String queue) { + QueuePath queue) { return warnings.stream().filter((w) -> w.getWarningType().equals(warningTypeToSelect) - && w.getQueue().equals(queue)).findFirst(); + && w.getQueue().equals(queue.getFullPath())).findFirst(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUniformQueueResourceCalculation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUniformQueueResourceCalculation.java index 863baaaaf95fb..6125bd30d4941 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUniformQueueResourceCalculation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUniformQueueResourceCalculation.java @@ -146,12 +146,12 @@ public void testPercentageResourceCalculation() throws IOException { @Test public void testAbsoluteResourceCalculation() throws IOException { - csConf.setMinimumResourceRequirement("", new QueuePath(A), QUEUE_A_RES); - csConf.setMinimumResourceRequirement("", new QueuePath(B), QUEUE_B_RES); - csConf.setMinimumResourceRequirement("", new QueuePath(A1), QUEUE_A1_RES); - csConf.setMinimumResourceRequirement("", new QueuePath(A2), QUEUE_A2_RES); - csConf.setMinimumResourceRequirement("", new QueuePath(A11), QUEUE_A11_RES); - csConf.setMinimumResourceRequirement("", new QueuePath(A12), QUEUE_A12_RES); + csConf.setMinimumResourceRequirement("", A, QUEUE_A_RES); + csConf.setMinimumResourceRequirement("", B, QUEUE_B_RES); + csConf.setMinimumResourceRequirement("", A1, QUEUE_A1_RES); + csConf.setMinimumResourceRequirement("", A2, QUEUE_A2_RES); + csConf.setMinimumResourceRequirement("", A11, QUEUE_A11_RES); + csConf.setMinimumResourceRequirement("", A12, QUEUE_A12_RES); QueueAssertionBuilder queueAssertionBuilder = createAssertionBuilder() .withQueue(A) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java index 6f1ff352c357b..1dabe53f12b6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java @@ -85,8 +85,7 @@ public void testPercentageCapacityConfig() { Assert.assertEquals(PERCENTAGE_VALUE, vcore.getResourceValue(), EPSILON); QueueCapacityVector rootCapacityVector = - capacityConfigParser.parse(Float.toString(PERCENTAGE_VALUE), - CapacitySchedulerConfiguration.ROOT); + capacityConfigParser.parse(Float.toString(PERCENTAGE_VALUE), ROOT); QueueCapacityVectorEntry memoryRoot = rootCapacityVector.getResource(MEMORY_URI); QueueCapacityVectorEntry vcoreRoot = rootCapacityVector.getResource(VCORES_URI); From 3e4f39bf7484bc4f3177f27202dcf9ee03d40a45 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Wed, 1 Mar 2023 14:25:01 +0100 Subject: [PATCH 09/21] YARN-11041. Unit test fixes Change-Id: I8be239b799bd04461d8b7e4ccc04b395a7666d8d --- .../resourcemanager/webapp/dao/CapacitySchedulerInfo.java | 4 +--- .../webapp/dao/CapacitySchedulerQueueInfo.java | 3 +-- .../capacity/TestApplicationLimitsByPartition.java | 6 +++--- .../resourcemanager/scheduler/capacity/TestLeafQueue.java | 7 +++---- .../scheduler/capacity/TestQueueStateManager.java | 2 +- .../yarn/server/resourcemanager/webapp/TestRMWebApp.java | 4 ++-- .../webapp/TestRMWebServiceAppsNodelabel.java | 4 ++-- .../webapp/TestRMWebServicesConfigurationMutation.java | 4 ++-- 8 files changed, 15 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 35668ba3054a3..91fb02323ede5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -50,7 +50,6 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected float normalizedWeight; protected String queueName; private String queuePath; - private QueuePath queuePathObject; protected int maxParallelApps; private boolean isAbsoluteResource; protected CapacitySchedulerQueueInfoList queues; @@ -78,7 +77,6 @@ public CapacitySchedulerInfo() { public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { this.queueName = parent.getQueueName(); this.queuePath = parent.getQueuePath(); - this.queuePathObject = parent.getQueuePathObject(); this.usedCapacity = parent.getUsedCapacity() * 100; this.capacity = parent.getCapacity() * 100; float max = parent.getMaximumCapacity(); @@ -100,7 +98,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { CapacitySchedulerConfiguration conf = cs.getConfiguration(); queueAcls = new QueueAclsInfo(); - queueAcls.addAll(getSortedQueueAclInfoList(parent, queuePathObject, conf)); + queueAcls.addAll(getSortedQueueAclInfoList(parent, new QueuePath(queuePath), conf)); queuePriority = parent.getPriority().getPriority(); if (parent instanceof ParentQueue) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index f6817ac35666e..f77914b06a52a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -55,7 +55,6 @@ public class CapacitySchedulerQueueInfo { static final float EPSILON = 1e-8f; protected String queuePath; - protected QueuePath queuePathObject; protected float capacity; protected float usedCapacity; protected float maxCapacity; @@ -104,7 +103,7 @@ public class CapacitySchedulerQueueInfo { CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) { queuePath = q.getQueuePath(); - queuePathObject = q.getQueuePathObject(); + QueuePath queuePathObject = new QueuePath(queuePath); capacity = q.getCapacity() * 100; usedCapacity = q.getUsedCapacity() * 100; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index b65b95cf7b6dd..6e8c0b02f9473 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -78,8 +78,8 @@ public class TestApplicationLimitsByPartition { final static int GB = 1024; final static String A1_PATH = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; - final static String B1_PATH = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1"; - final static String B2_PATH = CapacitySchedulerConfiguration.ROOT + ".a" + ".b2"; + final static String B1_PATH = CapacitySchedulerConfiguration.ROOT + ".b" + ".b1"; + final static String B2_PATH = CapacitySchedulerConfiguration.ROOT + ".b" + ".b2"; final static String C1_PATH = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1"; final static QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); final static QueuePath A1 = new QueuePath(A1_PATH); @@ -942,7 +942,7 @@ public void testAMLimitByAllResources() throws Exception { String queueName = "a1"; csconf.setQueues(ROOT, new String[] {queueName}); - csconf.setCapacity(A1, 100); + csconf.setCapacity(new QueuePath("root.a1"), 100); ResourceInformation res0 = ResourceInformation.newInstance("memory-mb", ResourceInformation.MEMORY_MB.getUnits(), GB, Long.MAX_VALUE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 3fa5cfeabad53..ba4215dfd79e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -264,9 +264,8 @@ private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels) private static final String C1 = "c1"; private static final String D = "d"; private static final String E = "e"; - final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); - private static final QueuePath A_QUEUE_PATH = - new QueuePath(CapacitySchedulerConfiguration.ROOT + A); + private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A_QUEUE_PATH = ROOT.createNewLeaf(A); private void setupQueueConfiguration( CapacitySchedulerConfiguration conf, final String newRootName, boolean withNodeLabels) { @@ -309,7 +308,7 @@ private void setupQueueConfiguration( conf.setAcl(b, QueueACL.SUBMIT_APPLICATIONS, "*"); final String cPath = newRootPath + "." + C; - final QueuePath c = new QueuePath(newRootPath); + final QueuePath c = new QueuePath(cPath); conf.setCapacity(c, 1.5f); conf.setMaximumCapacity(c, 10); conf.setAcl(c, QueueACL.SUBMIT_APPLICATIONS, " "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java index ec26a3d338b49..14a3089291900 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java @@ -56,7 +56,7 @@ public class TestQueueStateManager { private final static QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); private final static QueuePath Q1_QUEUE_PATH = new QueuePath(Q1_PATH); private final static QueuePath Q2_QUEUE_PATH = new QueuePath(Q2_PATH); - private final static QueuePath Q3_QUEUE_PATH = new QueuePath(Q1_PATH); + private final static QueuePath Q3_QUEUE_PATH = new QueuePath(Q3_PATH); private CapacityScheduler cs; private YarnConfiguration conf; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 18c86ae8f79d9..eef5c22539ef6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -339,7 +339,7 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf, final QueuePath B = root.createNewLeaf("b"); conf.setCapacity(B, 20); - final QueuePath C = root.createNewLeaf("v"); + final QueuePath C = root.createNewLeaf("c"); conf.setCapacity(C, 70); // Define 2nd-level queues @@ -360,7 +360,7 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf, final QueuePath C1 = C.createNewLeaf("c1"); final QueuePath C2 = C.createNewLeaf("c2"); final QueuePath C3 = C.createNewLeaf("c3"); - final QueuePath C4 = C.createNewLeaf("c3"); + final QueuePath C4 = C.createNewLeaf("c4"); conf.setQueues(C, new String[] {"c1", "c2", "c3", "c4"}); conf.setCapacity(C1, 50); conf.setCapacity(C2, 10); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java index 39e68974d1672..422a9bb31a4cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServiceAppsNodelabel.java @@ -118,8 +118,8 @@ private static void setupQueueConfiguration( // Define top-level queues QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); - QueuePath queueA = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".a"); - QueuePath defaultQueue = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".default"); + QueuePath queueA = root.createNewLeaf("a"); + QueuePath defaultQueue = root.createNewLeaf("default"); config.setQueues(root, new String[]{"a", "default"}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java index 9cec010dde62a..151dbb4b09f4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java @@ -603,7 +603,7 @@ public void testRemoveParentQueue() throws Exception { CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); assertEquals(3, newCSConf.getQueues(ROOT).size()); - assertNull(newCSConf.getQueues(ROOT_C)); + assertEquals(0, newCSConf.getQueues(ROOT_C).size()); } @Test @@ -633,7 +633,7 @@ public void testRemoveParentQueueWithCapacity() throws Exception { assertEquals(Status.OK.getStatusCode(), response.getStatus()); CapacitySchedulerConfiguration newCSConf = ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration(); - assertEquals(3, newCSConf.getQueues(ROOT_B).size()); + assertEquals(3, newCSConf.getQueues(ROOT).size()); assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.01f); } From b0d36f0627b7d8d3e859f0c3d9b7b24acffefa88 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Wed, 1 Mar 2023 15:28:37 +0100 Subject: [PATCH 10/21] YARN-11041. Checkstyle fixes Change-Id: I78659093ec68af98ca411cfca4bb48ba219fb323 --- .../capacity/AutoCreatedQueueTemplate.java | 2 - .../CapacitySchedulerQueueManager.java | 2 - .../scheduler/capacity/QueuePath.java | 4 +- .../scheduler/capacity/QueuePrefixes.java | 17 +++++ .../conf/ConfigurationUpdateAssembler.java | 2 - .../conf/QueueCapacityConfigParser.java | 2 - .../webapp/dao/LeafQueueTemplateInfo.java | 1 - .../server/resourcemanager/ACLsTestBase.java | 8 +-- .../ReservationACLsTestBase.java | 1 - .../TestApplicationMasterServiceCapacity.java | 40 +++++------ .../TestWorkPreservingRMRestart.java | 6 +- ...tProportionalCapacityPreemptionPolicy.java | 4 +- .../nodelabels/TestRMNodeLabelsManager.java | 10 +-- .../capacity/TestApplicationLimits.java | 20 +++--- ...tySchedulerAutoCreatedQueuePreemption.java | 2 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 14 ++-- .../TestCapacitySchedulerOvercommit.java | 2 +- ...stCapacitySchedulerSurgicalPreemption.java | 16 ++--- .../TestNodeLabelContainerAllocation.java | 15 ++-- .../scheduler/capacity/TestQueueMappings.java | 2 +- .../scheduler/capacity/TestQueueParsing.java | 20 +++--- .../capacity/TestQueueStateManager.java | 3 +- .../scheduler/capacity/TestReservations.java | 4 +- .../conf/TestQueueCapacityConfigParser.java | 3 +- .../scheduler/fifo/TestFifoScheduler.java | 2 +- .../resourcemanager/webapp/TestRMWebApp.java | 72 +++++++++---------- ...WebServicesCapacitySchedDynamicConfig.java | 1 - .../TestRMWebServicesForCSWithPartitions.java | 3 +- .../webapp/BaseRouterWebServicesTest.java | 3 +- .../TestableFederationInterceptorREST.java | 1 - 30 files changed, 142 insertions(+), 140 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index 003bcacd08508..d31f4307cc73d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -25,9 +25,7 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.util.Lists; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 8c4623f33187f..f4a87e86c5ed6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -20,8 +20,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index 8d0c910cb44cb..bbc460733d45e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -65,7 +65,7 @@ public QueuePath(String fullPath) { /** * Creates a new {@code QueuePath} with the current object as a parent, and appends the - * child to its queue path + * child to its queue path. * @param childName Name of the child which will be added to the queue path * @return QueuePath object */ @@ -256,7 +256,7 @@ public List getWildcardedQueuePaths(int maxAutoCreatedQueueDepth) { } /** - * Returns the supported wildcard level for this queue path + * Returns the supported wildcard level for this queue path. * @param maxAutoCreatedQueueDepth the maximum depth of auto-created queues stored in the * configuration * @return int value of the supported wildcard level diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java index e5fdabe790919..be31b23d66679 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java index 1f286ed1edccf..f0d18bbb594aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java @@ -20,7 +20,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; @@ -28,7 +27,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java index 71e4bfb404818..a0945b9dc4870 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java @@ -18,11 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.ResourceUnitCapacityType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.util.UnitsConversionUtil; import java.util.ArrayList; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java index 98112012b7ec9..c4a20494bc38b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java index 00591fe1b532b..26e3ee2920d0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java @@ -49,12 +49,12 @@ public abstract class ACLsTestBase { protected static final String QUEUEA = "queueA"; protected static final String QUEUEB = "queueB"; protected static final String QUEUEC = "queueC"; - protected static QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); - protected static QueuePath A_QUEUE_PATH = new QueuePath( + protected static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + protected static final QueuePath A_QUEUE_PATH = new QueuePath( CapacitySchedulerConfiguration.ROOT + "." + QUEUEA); - protected static QueuePath B_QUEUE_PATH = new QueuePath( + protected static final QueuePath B_QUEUE_PATH = new QueuePath( CapacitySchedulerConfiguration.ROOT + "." + QUEUEB); - protected static QueuePath C_QUEUE_PATH = new QueuePath( + protected static final QueuePath C_QUEUE_PATH = new QueuePath( CapacitySchedulerConfiguration.ROOT + "." + QUEUEC); protected static final Logger LOG = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java index 91edc92947819..6d5668ed010ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java @@ -49,7 +49,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java index dd1f295758d9e..ab68515911651 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java @@ -261,26 +261,26 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config); // Define top-level queues - final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); - conf.setQueues(ROOT, new String[] {"a", "b"}); - conf.setCapacityByLabel(ROOT, "x", 100); - conf.setCapacityByLabel(ROOT, "y", 100); - - final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; - final QueuePath A = new QueuePath(A_PATH); - conf.setCapacity(A, 50); - conf.setMaximumCapacity(A, 100); - conf.setAccessibleNodeLabels(A, toSet("x")); - conf.setDefaultNodeLabelExpression(A, "x"); - conf.setCapacityByLabel(A, "x", 100); - - final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b"; - final QueuePath B = new QueuePath(B_PATH); - conf.setCapacity(B, 50); - conf.setMaximumCapacity(B, 100); - conf.setAccessibleNodeLabels(B, toSet("y")); - conf.setDefaultNodeLabelExpression(B, "y"); - conf.setCapacityByLabel(B, "y", 100); + final QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); + conf.setQueues(root, new String[] {"a", "b"}); + conf.setCapacityByLabel(root, "x", 100); + conf.setCapacityByLabel(root, "y", 100); + + final String aPath = CapacitySchedulerConfiguration.ROOT + ".a"; + final QueuePath a = new QueuePath(aPath); + conf.setCapacity(a, 50); + conf.setMaximumCapacity(a, 100); + conf.setAccessibleNodeLabels(a, toSet("x")); + conf.setDefaultNodeLabelExpression(a, "x"); + conf.setCapacityByLabel(a, "x", 100); + + final String bPath = CapacitySchedulerConfiguration.ROOT + ".b"; + final QueuePath b = new QueuePath(bPath); + conf.setCapacity(b, 50); + conf.setMaximumCapacity(b, 100); + conf.setAccessibleNodeLabels(b, toSet("y")); + conf.setDefaultNodeLabelExpression(b, "y"); + conf.setCapacityByLabel(b, "y", 100); return conf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 2bbdbb5ec0230..0485888895026 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -582,7 +582,7 @@ private void checkFSQueue(ResourceManager rm, private static final QueuePath B2_QUEUE_PATH = new QueuePath(Q_B2_PATH); private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { - conf.setQueues(ROOT, new String[] { R }); + conf.setQueues(ROOT, new String[] {R}); conf.setCapacity(R_QUEUE_PATH, 100); conf.setQueues(R_QUEUE_PATH, new String[] {A, B}); conf.setCapacity(A_QUEUE_PATH, 50); @@ -593,7 +593,7 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { private void setupQueueConfigurationOnlyA( CapacitySchedulerConfiguration conf) { - conf.setQueues(ROOT, new String[] { R }); + conf.setQueues(ROOT, new String[] {R}); conf.setCapacity(R_QUEUE_PATH, 10); conf.setQueues(R_QUEUE_PATH, new String[] {A}); conf.setCapacity(A_QUEUE_PATH, 100); @@ -602,7 +602,7 @@ private void setupQueueConfigurationOnlyA( } private void setupQueueConfigurationChildOfB(CapacitySchedulerConfiguration conf) { - conf.setQueues(ROOT, new String[] { R }); + conf.setQueues(ROOT, new String[] {R}); conf.setCapacity(R_QUEUE_PATH, 100); conf.setQueues(R_QUEUE_PATH, new String[] {A, B}); conf.setCapacity(A_QUEUE_PATH, 50); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index 5bf7685770d42..1bac22743b0ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -133,8 +133,8 @@ public class TestProportionalCapacityPreemptionPolicy { private static final QueuePath QUEUE_A_QUEUE_B = new QueuePath("root.queueA.queueB"); private static final QueuePath QUEUE_B = new QueuePath("root.queueB"); private static final QueuePath QUEUE_D = new QueuePath("root.queueD"); - private static final QueuePath QUEUE_D_QUEUE_E = new QueuePath( "root.queueD.queueE"); - private static final QueuePath QUEUE_A_QUEUE_C = new QueuePath( "root.queueA.queueC"); + private static final QueuePath QUEUE_D_QUEUE_E = new QueuePath("root.queueD.queueE"); + private static final QueuePath QUEUE_A_QUEUE_C = new QueuePath("root.queueA.queueC"); public enum priority { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index bfd3b0ca6c10d..affd984e5334a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -669,13 +669,13 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { new CapacitySchedulerConfiguration(config); // Define top-level queues final QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); - conf.setQueues(root, new String[] { "a" }); + conf.setQueues(root, new String[] {"a"}); conf.setCapacityByLabel(root, "x", 100); - final QueuePath A = root.createNewLeaf("a"); - conf.setCapacity(A, 100); - conf.setAccessibleNodeLabels(A, ImmutableSet.of("x")); - conf.setCapacityByLabel(A, "x", 100); + final QueuePath a = root.createNewLeaf("a"); + conf.setCapacity(a, 100); + conf.setAccessibleNodeLabels(a, ImmutableSet.of("x")); + conf.setCapacityByLabel(a, "x", 100); return conf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 10417f8a61e47..d7a66a1407ec3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -149,17 +149,15 @@ public void setUp() throws IOException { private static final String AA1 = "a1"; private static final String AA2 = "a2"; private static final String AA3 = "a3"; - private static final QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); - private static final QueuePath A_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT, A); - private static final QueuePath B_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT, B); - private static final QueuePath C_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT, C); - private static final QueuePath D_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT, D); - private static final QueuePath AA1_QUEUE_PATH = QueuePath.createFromQueues( - CapacitySchedulerConfiguration.ROOT, A, AA1); - private static final QueuePath AA2_QUEUE_PATH = QueuePath.createFromQueues( - CapacitySchedulerConfiguration.ROOT, A, AA2); - private static final QueuePath AA3_QUEUE_PATH = QueuePath.createFromQueues( - CapacitySchedulerConfiguration.ROOT, A, AA3); + private static final QueuePath ROOT_QUEUE_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath A_QUEUE_PATH = ROOT_QUEUE_PATH.createNewLeaf(A); + private static final QueuePath B_QUEUE_PATH = ROOT_QUEUE_PATH.createNewLeaf(B); + private static final QueuePath C_QUEUE_PATH = ROOT_QUEUE_PATH.createNewLeaf(C); + private static final QueuePath D_QUEUE_PATH = ROOT_QUEUE_PATH.createNewLeaf(D); + private static final QueuePath AA1_QUEUE_PATH = A_QUEUE_PATH.createNewLeaf(AA1); + private static final QueuePath AA2_QUEUE_PATH = A_QUEUE_PATH.createNewLeaf(AA2); + private static final QueuePath AA3_QUEUE_PATH = A_QUEUE_PATH.createNewLeaf(AA3); private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java index 4a3b5d9564d3d..09724b7d8d6ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java @@ -62,7 +62,7 @@ public void setUp() throws Exception { //setup new queues with one of them auto enabled // Define top-level queues // Set childQueue for root - conf.setQueues(ROOT, new String[] { "c" }); + conf.setQueues(ROOT, new String[] {"c"}); conf.setCapacity(C, 100f); conf.setUserLimitFactor(C, 1.0f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index 8f2b7f20fc08f..1f880c87ad22e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -85,13 +85,13 @@ private Configuration getConfigurationWithQueueLabels(Configuration config) { conf.setCapacityByLabel(ROOT, "y", 100); conf.setCapacityByLabel(ROOT, "z", 100); - final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; - final QueuePath A = new QueuePath(A_PATH); - conf.setCapacity(A, 100); - conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z")); - conf.setCapacityByLabel(A, "x", 100); - conf.setCapacityByLabel(A, "y", 100); - conf.setCapacityByLabel(A, "z", 100); + final String aPath = CapacitySchedulerConfiguration.ROOT + ".a"; + final QueuePath a = new QueuePath(aPath); + conf.setCapacity(a, 100); + conf.setAccessibleNodeLabels(a, ImmutableSet.of("x", "y", "z")); + conf.setCapacityByLabel(a, "x", 100); + conf.setCapacityByLabel(a, "y", 100); + conf.setCapacityByLabel(a, "z", 100); return conf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java index e3051165cef31..4a2a10f6032eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java @@ -31,7 +31,7 @@ public class TestCapacitySchedulerOvercommit extends TestSchedulerOvercommit { private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); private static final QueuePath DEFAULT = new QueuePath(CapacitySchedulerConfiguration.ROOT - + ".default"); + + ".default"); @Override protected Configuration getConfiguration() { Configuration conf = super.getConfiguration(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java index ccef92fc156ad..322b8f5b6cb36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java @@ -761,7 +761,7 @@ public void testPriorityPreemptionFromHighestPriorityQueueAndOldestContainer() */ // A/B has higher priority - conf.setQueuePriority(A , 1); + conf.setQueuePriority(A, 1); conf.setQueuePriority(B, 2); conf.setCapacity(A, 45f); conf.setCapacity(B, 45f); @@ -998,14 +998,14 @@ protected RMNodeLabelsManager createNodeLabelManager() { private void initializeConfProperties(CapacitySchedulerConfiguration conf) throws IOException { - QueuePath A_QUEUE_PATH = new QueuePath("root.A"); - QueuePath B_QUEUE_PATH = new QueuePath("root.B"); + QueuePath aQueuePath = new QueuePath("root.A"); + QueuePath bQueuePath = new QueuePath("root.B"); conf.setQueues(ROOT, new String[] {"A", "B"}); - conf.setCapacity(A_QUEUE_PATH, 50); - conf.setCapacity(B_QUEUE_PATH, 50); - conf.setQueuePriority(A_QUEUE_PATH, 1); - conf.setQueuePriority(B_QUEUE_PATH, 2); + conf.setCapacity(aQueuePath, 50); + conf.setCapacity(bQueuePath, 50); + conf.setQueuePriority(aQueuePath, 1); + conf.setQueuePriority(bQueuePath, 2); conf.set(PREFIX + "root.ordering-policy", "priority-utilization"); conf.set(PREFIX + "ordering-policy.priority-utilization.underutilized-preemption.enabled", "true"); @@ -1184,7 +1184,7 @@ public void testPreemptionForFragmentatedCluster() throws Exception { this.conf); conf.setLong(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024 * 21); - conf.setQueues(ROOT, new String[] { "a", "b" }); + conf.setQueues(ROOT, new String[] {"a", "b"}); conf.setCapacity(A, 50); conf.setUserLimitFactor(A, 100); conf.setCapacity(B, 50); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java index 19406c91252fd..67a4113d675c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java @@ -1730,8 +1730,7 @@ public RMNodeLabelsManager createNodeLabelManager() { new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(ROOT, new String[] { "a", - "b" }); + csConf.setQueues(ROOT, new String[] {"a", "b"}); csConf.setCapacityByLabel(ROOT, "x", 100); csConf.setCapacity(A, 50); @@ -1880,8 +1879,7 @@ public void testQueueUsedCapacitiesUpdate() new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(ROOT, new String[] { "a", - "b" }); + csConf.setQueues(ROOT, new String[] {"a", "b"}); csConf.setCapacityByLabel(ROOT, "x", 100); /** @@ -2137,8 +2135,7 @@ public void testOrderOfAllocationOnPartitions() new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(ROOT, new String[] { "a", - "b", "c", "d" }); + csConf.setQueues(ROOT, new String[] {"a", "b", "c", "d"}); csConf.setCapacityByLabel(ROOT, "x", 100); csConf.setCapacity(A, 25); @@ -2420,8 +2417,7 @@ public void testParentQueueMaxCapsAreRespected() throws Exception { new CapacitySchedulerConfiguration(this.conf); // Define top-level queues - csConf.setQueues(ROOT, new String[] { "a", - "b"}); + csConf.setQueues(ROOT, new String[] {"a", "b"}); csConf.setCapacityByLabel(ROOT, "x", 100); csConf.setCapacity(A, 10); @@ -3305,8 +3301,7 @@ public void testQueueMetricsWithLabelsDisableElasticity() throws Exception { csConf.setMaximumCapacityByLabel(B, "x", 100); // Define 2nd-level queues - csConf.setQueues(A, new String[] { "a1", - "a2"}); + csConf.setQueues(A, new String[] {"a1", "a2"}); csConf.setCapacity(A1, 20); csConf.setMaximumCapacity(A1, 60); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java index a49f155b87d06..ce31ea71af455 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java @@ -66,7 +66,7 @@ public void setup() { private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(ROOT, new String[] { Q1, Q2 }); + conf.setQueues(ROOT, new String[] {Q1, Q2}); conf.setCapacity(Q1_QUEUE_PATH, 10); conf.setCapacity(Q2_QUEUE_PATH, 90); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 22414e80b2d47..017f9d4f56c7f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -134,7 +134,7 @@ private void setupQueueConfigurationWithSpacesShouldBeTrimmed( CapacitySchedulerConfiguration conf) { // Define top-level queues conf.set( - QueuePrefixes + QueuePrefixes .getQueuePrefix(ROOT) + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); @@ -151,7 +151,7 @@ private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed( CapacitySchedulerConfiguration conf) { // Define top-level queues conf.set( - QueuePrefixes + QueuePrefixes .getQueuePrefix(ROOT) + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); @@ -278,8 +278,8 @@ public void testMaxCapacity() throws Exception { fail = false; try { - LeafQueue a = (LeafQueue)capacityScheduler.getQueue(A_PATH); - a.setMaxCapacity(45); + LeafQueue a = (LeafQueue)capacityScheduler.getQueue(A_PATH); + a.setMaxCapacity(45); } catch (IllegalArgumentException iae) { fail = true; } @@ -1021,8 +1021,8 @@ public void testQueueParsingWithMoveQueue() YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); - csConf.setQueues(ROOT, new String[] { "a" }); - csConf.setQueues(A, new String[] { "x", "y" }); + csConf.setQueues(ROOT, new String[] {"a"}); + csConf.setQueues(A, new String[] {"x", "y"}); csConf.setCapacity(A, 100); csConf.setCapacity(AX, 50); csConf.setCapacity(AY, 50); @@ -1039,8 +1039,8 @@ public void testQueueParsingWithMoveQueue() capacityScheduler.init(csConf); capacityScheduler.start(); - csConf.setQueues(ROOT, new String[] { "a", "x" }); - csConf.setQueues(A, new String[] { "y" }); + csConf.setQueues(ROOT, new String[] {"a", "x"}); + csConf.setQueues(A, new String[] {"y"}); csConf.setCapacity(X, 50); csConf.setCapacity(A, 50); csConf.setCapacity(AY, 100); @@ -1059,7 +1059,7 @@ public void testRMStartWrongNodeCapacity() throws Exception { new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(ROOT, new String[] { "a" }); + conf.setQueues(ROOT, new String[] {"a"}); conf.setCapacityByLabel(ROOT, "x", 100); conf.setCapacityByLabel(ROOT, "y", 100); conf.setCapacityByLabel(ROOT, "z", 100); @@ -1135,7 +1135,7 @@ public void testQueueCapacityWithWeight() throws Exception { new CapacitySchedulerConfiguration(config); // Define top-level queues - conf.setQueues(ROOT, new String[] { "a" }); + conf.setQueues(ROOT, new String[] {"a"}); conf.setLabeledQueueWeight(ROOT, "x", 100); conf.setLabeledQueueWeight(ROOT, "y", 100); conf.setLabeledQueueWeight(ROOT, "z", 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java index 14a3089291900..27ae3c3ae0187 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueStateManager.java @@ -53,7 +53,8 @@ public class TestQueueStateManager { Q1_PATH + "." + Q2; private final static String Q3_PATH = Q1_PATH + "." + Q3; - private final static QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private final static QueuePath ROOT_QUEUE_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT); private final static QueuePath Q1_QUEUE_PATH = new QueuePath(Q1_PATH); private final static QueuePath Q2_QUEUE_PATH = new QueuePath(Q2_PATH); private final static QueuePath Q3_QUEUE_PATH = new QueuePath(Q3_PATH); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index 9037be4e495a9..5240362c239d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -172,11 +172,11 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf, QueuePath newRootPath = root.createNewLeaf(newRoot); QueuePath aQueuePath = newRootPath.createNewLeaf(A); - conf.setQueues(root, new String[] { newRoot }); + conf.setQueues(root, new String[] {newRoot}); conf.setMaximumCapacity(root, 100); conf.setAcl(root, QueueACL.SUBMIT_APPLICATIONS, " "); - conf.setQueues(newRootPath, new String[] { A }); + conf.setQueues(newRootPath, new String[] {A}); conf.setCapacity(newRootPath, 100); conf.setMaximumCapacity(newRootPath, 100); conf.setAcl(newRootPath, QueueACL.SUBMIT_APPLICATIONS, " "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java index 1dabe53f12b6b..9025030884229 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java @@ -115,7 +115,8 @@ public void testWeightCapacityConfig() { @Test public void testAbsoluteCapacityVectorConfig() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE); + conf.set(getQueuePrefix(QUEUE_PATH) + CapacitySchedulerConfiguration.CAPACITY, + ABSOLUTE_RESOURCE); conf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_TYPES); ResourceUtils.resetResourceTypes(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 00fe3dcd21045..682a01d985fa4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -862,7 +862,7 @@ public void testNonDefaultMinimumAllocation() throws Exception { public void testReconnectedNode() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); QueuePath defaultQueuePath = new QueuePath("default"); - conf.setQueues(defaultQueuePath, new String[] { "default" }); + conf.setQueues(defaultQueuePath, new String[] {"default"}); conf.setCapacity(defaultQueuePath, 100); FifoScheduler fs = new FifoScheduler(); fs.init(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index eef5c22539ef6..93d4e695605fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -333,48 +333,48 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf, QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT); conf.setQueues(root, new String[] {"a", "b", "c"}); - final QueuePath A = root.createNewLeaf("a"); - conf.setCapacity(A, 10); + final QueuePath a = root.createNewLeaf("a"); + conf.setCapacity(a, 10); - final QueuePath B = root.createNewLeaf("b"); - conf.setCapacity(B, 20); + final QueuePath b = root.createNewLeaf("b"); + conf.setCapacity(b, 20); - final QueuePath C = root.createNewLeaf("c"); - conf.setCapacity(C, 70); + final QueuePath c = root.createNewLeaf("c"); + conf.setCapacity(c, 70); // Define 2nd-level queues - final QueuePath A1 = A.createNewLeaf("a1"); - final QueuePath A2 = A.createNewLeaf("a2"); - conf.setQueues(A, new String[] {"a1", "a2"}); - conf.setCapacity(A1, 30); - conf.setCapacity(A2, 70); - - final QueuePath B1 = B.createNewLeaf("b1"); - final QueuePath B2 = B.createNewLeaf("b2"); - final QueuePath B3 = B.createNewLeaf("b3"); - conf.setQueues(B, new String[] {"b1", "b2", "b3"}); - conf.setCapacity(B1, 50); - conf.setCapacity(B2, 30); - conf.setCapacity(B3, 20); - - final QueuePath C1 = C.createNewLeaf("c1"); - final QueuePath C2 = C.createNewLeaf("c2"); - final QueuePath C3 = C.createNewLeaf("c3"); - final QueuePath C4 = C.createNewLeaf("c4"); - conf.setQueues(C, new String[] {"c1", "c2", "c3", "c4"}); - conf.setCapacity(C1, 50); - conf.setCapacity(C2, 10); - conf.setCapacity(C3, 35); - conf.setCapacity(C4, 5); + final QueuePath a1 = a.createNewLeaf("a1"); + final QueuePath a2 = a.createNewLeaf("a2"); + conf.setQueues(a, new String[] {"a1", "a2"}); + conf.setCapacity(a1, 30); + conf.setCapacity(a2, 70); + + final QueuePath b1 = b.createNewLeaf("b1"); + final QueuePath b2 = b.createNewLeaf("b2"); + final QueuePath b3 = b.createNewLeaf("b3"); + conf.setQueues(b, new String[] {"b1", "b2", "b3"}); + conf.setCapacity(b1, 50); + conf.setCapacity(b2, 30); + conf.setCapacity(b3, 20); + + final QueuePath c1 = c.createNewLeaf("c1"); + final QueuePath c2 = c.createNewLeaf("c2"); + final QueuePath c3 = c.createNewLeaf("c3"); + final QueuePath c4 = c.createNewLeaf("c4"); + conf.setQueues(c, new String[] {"c1", "c2", "c3", "c4"}); + conf.setCapacity(c1, 50); + conf.setCapacity(c2, 10); + conf.setCapacity(c3, 35); + conf.setCapacity(c4, 5); // Define 3rd-level queues - final QueuePath C11 = C1.createNewLeaf("c11"); - final QueuePath C12 = C1.createNewLeaf("c12"); - final QueuePath C13 = C1.createNewLeaf("c13"); - conf.setQueues(C1, new String[] {"c11", "c12", "c13"}); - conf.setCapacity(C11, 15); - conf.setCapacity(C12, 45); - conf.setCapacity(C13, 40); + final QueuePath c11 = c1.createNewLeaf("c11"); + final QueuePath c12 = c1.createNewLeaf("c12"); + final QueuePath c13 = c1.createNewLeaf("c13"); + conf.setQueues(c1, new String[] {"c11", "c12", "c13"}); + conf.setCapacity(c11, 15); + conf.setCapacity(c12, 45); + conf.setCapacity(c13, 40); if (useDRC) { conf.set("yarn.scheduler.capacity.resource-calculator", "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 2d512abe87ae9..665c8a92b5af2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; -import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.junit.Test; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index bfce233a0994a..67977920bff06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -100,7 +100,8 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase { private static final String QUEUE_A = "Qa"; private static final String LABEL_LY = "Ly"; private static final String LABEL_LX = "Lx"; - private static final QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath ROOT_QUEUE_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT); private static final ImmutableSet CLUSTER_LABELS = ImmutableSet.of(LABEL_LX, LABEL_LY, DEFAULT_PARTITION); private static final String DOT = "."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java index 6f55449265fe4..46508d91ed859 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java @@ -82,7 +82,8 @@ public abstract class BaseRouterWebServicesTest { public static final String QUEUE_DEDICATED = "dedicated"; public static final String QUEUE_DEDICATED_FULL = CapacitySchedulerConfiguration.ROOT + CapacitySchedulerConfiguration.DOT + QUEUE_DEDICATED; - public static final QueuePath ROOT_QUEUE_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); + public static final QueuePath ROOT_QUEUE_PATH = + new QueuePath(CapacitySchedulerConfiguration.ROOT); public static final QueuePath DEFAULT_QUEUE_PATH = new QueuePath(QUEUE_DEFAULT_FULL); public static final QueuePath DEDICATED_QUEUE_PATH = new QueuePath(QUEUE_DEDICATED_FULL); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java index 61fb43d861e89..4bf5950e2f52d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java @@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From f789293d97aa697b61d320b56a01a6dc0574b3dd Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Fri, 3 Mar 2023 14:24:01 +0100 Subject: [PATCH 11/21] YARN-11041. Unit test fixes Change-Id: I95540b77da3983f82943000b32f8d1ede791761f --- .../resourcemanager/scheduler/capacity/QueuePrefixes.java | 4 ++-- .../yarn/server/resourcemanager/TestAppManager.java | 8 ++++---- .../resourcemanager/TestWorkPreservingRMRestart.java | 2 +- .../resourcemanager/scheduler/capacity/TestLeafQueue.java | 2 +- .../webapp/TestRMWebServicesForCSWithPartitions.java | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java index be31b23d66679..7bd9c324eb645 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java @@ -42,8 +42,8 @@ public static String getNodeLabelPrefix(QueuePath queuePath, String label) { } /** - * Get the auto created leaf queue's template configuration prefix - * Leaf queue's template capacities are configured at the parent queue + * Get the auto created leaf queue's template configuration prefix. + * Leaf queue's template capacities are configured at the parent queue. * * @param queuePath parent queue's path * @return Config prefix for leaf queue template configurations diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 8470cd48b8d1a..fa659c6b453d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -526,8 +526,8 @@ public void testFlexibleAutoCreatedQueuesWithSpecializedACLTemplatesAndDynamicPa "1w"); csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue", "user3"); - csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications", - "user3"); + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + + "acl_submit_applications", "user3"); MockRM newMockRM = new MockRM(csConf); @@ -617,8 +617,8 @@ public void testFlexibleAutoCreatedQueuesWithMixedCommonLeafACLTemplatesAndDynam "1w"); csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue", "user3"); - csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications", - "user3"); + csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + + "acl_submit_applications", "user3"); testFlexibleAQCDWithMixedTemplatesDynamicParentACLScenario(conf, csConf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 0485888895026..7d22ba19e189c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -594,7 +594,7 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { private void setupQueueConfigurationOnlyA( CapacitySchedulerConfiguration conf) { conf.setQueues(ROOT, new String[] {R}); - conf.setCapacity(R_QUEUE_PATH, 10); + conf.setCapacity(R_QUEUE_PATH, 100); conf.setQueues(R_QUEUE_PATH, new String[] {A}); conf.setCapacity(A_QUEUE_PATH, 100); conf.setDouble(CapacitySchedulerConfiguration diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index ba4215dfd79e4..dee97ae18cdf4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -479,7 +479,7 @@ public void testPolicyConfiguration() throws Exception { "testPolicyRoot" + System.currentTimeMillis(); OrderingPolicy comPol = - testConf.getAppOrderingPolicy(new QueuePath(tproot)); + testConf.getAppOrderingPolicy(new QueuePath(tproot)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 67977920bff06..39d7bd61e1ee6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -161,7 +161,7 @@ private static void setupQueueConfiguration( config.setMaximumCapacity(leafQueueAPath, 50); String leafQueueB = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_B; - QueuePath leafQueueBPath = new QueuePath(leafQueueA); + QueuePath leafQueueBPath = new QueuePath(leafQueueB); config.setCapacity(leafQueueBPath, 30); config.setMaximumCapacity(leafQueueBPath, 50); @@ -169,12 +169,12 @@ private static void setupQueueConfiguration( config.setMaximumCapacity(interMediateQueueCPath, 50); String leafQueueC1 = interMediateQueueC + "." + LEAF_QUEUE_C1; - QueuePath leafQueueC1Path = new QueuePath(leafQueueA); + QueuePath leafQueueC1Path = new QueuePath(leafQueueC1); config.setCapacity(leafQueueC1Path, 50); config.setMaximumCapacity(leafQueueC1Path, 60); String leafQueueC2 = interMediateQueueC + "." + LEAF_QUEUE_C2; - QueuePath leafQueueC2Path = new QueuePath(leafQueueA); + QueuePath leafQueueC2Path = new QueuePath(leafQueueC2); config.setCapacity(leafQueueC2Path, 50); config.setMaximumCapacity(leafQueueC2Path, 70); From e2e24f2acb4645e4be685d2fd56675abcf54da8c Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Mon, 6 Mar 2023 09:46:57 +0100 Subject: [PATCH 12/21] YARN-11041. Refactored ACQ template prefix generation Change-Id: I7394673085b5e1b9309c9f49f814558ec6568847 --- .../resourcemanager/scheduler/capacity/QueuePath.java | 10 ---------- .../scheduler/capacity/QueuePrefixes.java | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index bbc460733d45e..67e81a0c4b8c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -63,16 +63,6 @@ public QueuePath(String fullPath) { setFromFullPath(fullPath); } - /** - * Creates a new {@code QueuePath} with the current object as a parent, and appends the - * child to its queue path. - * @param childName Name of the child which will be added to the queue path - * @return QueuePath object - */ - public QueuePath createChild(String childName) { - return new QueuePath(getFullPath(), childName); - } - /** * Constructor to create Queue path from queue names. * The provided queue names will be concatenated by dots, giving a full queue path. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java index 7bd9c324eb645..3d91826c48ca5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java @@ -53,6 +53,6 @@ public static String getAutoCreatedQueueTemplateConfPrefix(QueuePath queuePath) } public static QueuePath getAutoCreatedQueueObjectTemplateConfPrefix(QueuePath queuePath) { - return queuePath.createChild(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX); + return new QueuePath(getAutoCreatedQueueTemplateConfPrefix(queuePath)); } } From afb669e2c3748e78102733164419502e9aeee694 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Mon, 6 Mar 2023 13:42:46 +0100 Subject: [PATCH 13/21] YARN-11041. Fixed imports Change-Id: I9d1c61101efcce976a822b1320dd4505e38c8728 --- .../capacity/TestCapacitySchedulerWeightMode.java | 2 -- .../router/webapp/MockDefaultRequestInterceptorREST.java | 7 ++++++- .../router/webapp/TestableFederationInterceptorREST.java | 6 +++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java index af9d07e40885f..216ec0f9b9261 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java @@ -51,8 +51,6 @@ import java.util.ArrayList; import java.util.Set; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.*; - public class TestCapacitySchedulerWeightMode { private static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default"; private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java index 3c4631860c8ca..879f1baef4d5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java @@ -162,7 +162,12 @@ import org.slf4j.LoggerFactory; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.*; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.DEDICATED_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.DEFAULT_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.ROOT_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEFAULT; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED_FULL; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java index 4bf5950e2f52d..5410bf59346a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java @@ -33,7 +33,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.*; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.DEFAULT_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.ROOT_QUEUE_PATH; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED_FULL; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEFAULT; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED; /** * Extends the FederationInterceptorREST and overrides methods to provide a From 5a6251dd5737de21f2f3ca5f384cefdb39807e8c Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Fri, 19 May 2023 17:58:09 +0200 Subject: [PATCH 14/21] YARN-11041. Resolved conflicts Change-Id: I2bd30982c32386d68dc662dc268773e0d56be777 --- .../scheduler/capacity/AbstractParentQueue.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java index 8a849d28a2eef..89ef1c41e51fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java @@ -133,7 +133,7 @@ public AbstractParentQueue(CapacitySchedulerQueueContext queueContext, this.childQueues = new ArrayList<>(); this.allowZeroCapacitySum = queueContext.getConfiguration() - .getAllowZeroCapacitySum(getQueuePath()); + .getAllowZeroCapacitySum(getQueuePathObject()); } @@ -167,7 +167,7 @@ protected void setupQueueConfigs(Resource clusterResource) // Initialize queue ordering policy queueOrderingPolicy = configuration.getQueueOrderingPolicy( - getQueuePath(), parent == null ? + getQueuePathObject(), parent == null ? null : ((AbstractParentQueue) parent).getQueueOrderingPolicyConfigName()); queueOrderingPolicy.setQueues(childQueues); @@ -549,7 +549,7 @@ public void removeChildQueue(CSQueue queue) */ public boolean isEligibleForAutoQueueCreation() { return isDynamicQueue() || queueContext.getConfiguration(). - isAutoQueueCreationV2Enabled(getQueuePath()); + isAutoQueueCreationV2Enabled(getQueuePathObject()); } @Override @@ -1585,7 +1585,7 @@ Map getEffectiveMinRatio(String label) { public boolean isEligibleForAutoDeletion() { return isDynamicQueue() && getChildQueues().size() == 0 && queueContext.getConfiguration(). - isAutoExpiredDeletionEnabled(this.getQueuePath()); + isAutoExpiredDeletionEnabled(this.getQueuePathObject()); } public AutoCreatedQueueTemplate getAutoCreatedQueueTemplate() { From 24821258d07ab8bac612d493858b5827121e5c5d Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Mon, 12 Jun 2023 14:11:33 +0200 Subject: [PATCH 15/21] YARN-11041. Review fixes Change-Id: I73c94199a8df7c910afac5ffa25257bfa2b991f6 --- .../CapacitySchedulerConfiguration.java | 5 +- .../scheduler/capacity/QueuePath.java | 5 +- .../scheduler/capacity/QueuePrefixes.java | 3 +- .../conf/ConfigurationUpdateAssembler.java | 78 +++++---- .../scheduler/capacity/TestQueuePath.java | 148 ++++++++++++++---- ...WebServicesCapacitySchedDynamicConfig.java | 4 +- .../TestableFederationInterceptorREST.java | 3 +- 7 files changed, 168 insertions(+), 78 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 6cd63c4d3d1c3..8e191d0fb0f70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1550,9 +1550,10 @@ public boolean isReservable(String queue) { } public void setReservable(String queue, boolean isReservable) { - setBoolean(getQueuePrefix(new QueuePath(queue)) + IS_RESERVABLE, isReservable); + QueuePath queuePath = new QueuePath(queue); + setBoolean(getQueuePrefix(queuePath) + IS_RESERVABLE, isReservable); LOG.debug("here setReservableQueue: queuePrefix={}, isReservableQueue={}", - getQueuePrefix(new QueuePath(queue)), isReservable(queue)); + getQueuePrefix(queuePath), isReservable(queue)); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java index 67e81a0c4b8c8..d832f50c65d09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java @@ -236,8 +236,9 @@ public List getWildcardedQueuePaths(int maxAutoCreatedQueueDepth) { int supportedWildcardLevel = getSupportedWildcardLevel(maxAutoCreatedQueueDepth); // Collect all template entries - for (int wildcardLevel = supportedWildcardLevel; wildcardLevel > 0; wildcardLevel--) { - pathComponents[wildcardLevel] = WILDCARD_QUEUE; + for (int wildcardLevel = 1; wildcardLevel <= supportedWildcardLevel; wildcardLevel++) { + int wildcardedComponentIndex = pathComponents.length - wildcardLevel; + pathComponents[wildcardedComponentIndex] = WILDCARD_QUEUE; QueuePath wildcardedPath = createFromQueues(pathComponents); wildcardedPaths.add(wildcardedPath); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java index 3d91826c48ca5..75b7f77a53394 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java @@ -30,8 +30,7 @@ private QueuePrefixes() { } public static String getQueuePrefix(QueuePath queuePath) { - String queueName = PREFIX + queuePath.getFullPath() + DOT; - return queueName; + return PREFIX + queuePath.getFullPath() + DOT; } public static String getNodeLabelPrefix(QueuePath queuePath, String label) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java index f0d18bbb594aa..7a0508d034923 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java @@ -22,6 +22,7 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; @@ -31,8 +32,6 @@ import java.util.List; import java.util.Map; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY; - public final class ConfigurationUpdateAssembler { private ConfigurationUpdateAssembler() { @@ -65,39 +64,34 @@ private static void removeQueue( if (queueToRemove == null) { return; } - if (queueToRemove.lastIndexOf('.') == -1) { - throw new IOException("Can't remove queue " + queueToRemove); + QueuePath queuePath = new QueuePath(queueToRemove); + if (queuePath.isRoot() || queuePath.isInvalid()) { + throw new IOException("Can't remove queue " + queuePath.getFullPath()); } - String queueName = queueToRemove.substring( - queueToRemove.lastIndexOf('.') + 1); - List siblingQueues = getSiblingQueues(queueToRemove, + String queueName = queuePath.getLeafName(); + List siblingQueues = getSiblingQueues(queuePath, proposedConf); if (!siblingQueues.contains(queueName)) { - throw new IOException("Queue " + queueToRemove + " not found"); + throw new IOException("Queue " + queuePath.getFullPath() + " not found"); } siblingQueues.remove(queueName); - String parentPath = queueToRemove.substring(0, queueToRemove - .lastIndexOf('.')); - QueuePath parentQueuePath = new QueuePath(parentPath); - proposedConf.setQueues(parentQueuePath, siblingQueues.toArray( + + QueuePath parentPath = queuePath.getParentObject(); + proposedConf.setQueues(parentPath, siblingQueues.toArray( new String[0])); - String queuesConfig = CapacitySchedulerConfiguration.PREFIX - + parentPath + CapacitySchedulerConfiguration.DOT - + CapacitySchedulerConfiguration.QUEUES; + String queuesConfig = getQueuesConfig(parentPath); if (siblingQueues.isEmpty()) { confUpdate.put(queuesConfig, null); // Unset Ordering Policy of Leaf Queue converted from // Parent Queue after removeQueue - String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX - + parentPath + CapacitySchedulerConfiguration.DOT - + ORDERING_POLICY; + String queueOrderingPolicy = getOrderingPolicyConfig(parentPath); proposedConf.unset(queueOrderingPolicy); confUpdate.put(queueOrderingPolicy, null); } else { confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues)); } for (Map.Entry confRemove : proposedConf.getValByRegex( - ".*" + queueToRemove + "\\..*") + ".*" + queuePath.getFullPath() + "\\..*") .entrySet()) { proposedConf.unset(confRemove.getKey()); confUpdate.put(confRemove.getKey(), null); @@ -110,26 +104,23 @@ private static void addQueue( if (addInfo == null) { return; } - String queuePath = addInfo.getQueue(); - String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1); - if (queuePath.lastIndexOf('.') == -1) { + QueuePath queuePath = new QueuePath(addInfo.getQueue()); + String queueName = queuePath.getLeafName(); + if (queuePath.isRoot() || queuePath.isInvalid()) { throw new IOException("Can't add invalid queue " + queuePath); } else if (getSiblingQueues(queuePath, proposedConf).contains( queueName)) { throw new IOException("Can't add existing queue " + queuePath); } - String parentPath = queuePath.substring(0, queuePath.lastIndexOf('.')); - QueuePath parentQueuePath = new QueuePath(parentPath); - List siblingQueues = proposedConf.getQueues(parentQueuePath); - siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1)); - proposedConf.setQueues(parentQueuePath, + + QueuePath parentPath = queuePath.getParentObject(); + List siblingQueues = proposedConf.getQueues(parentPath); + siblingQueues.add(queueName); + proposedConf.setQueues(parentPath, siblingQueues.toArray(new String[0])); - confUpdate.put(CapacitySchedulerConfiguration.PREFIX - + parentPath + CapacitySchedulerConfiguration.DOT - + CapacitySchedulerConfiguration.QUEUES, + confUpdate.put(getQueuesConfig(parentPath), Joiner.on(',').join(siblingQueues)); - String keyPrefix = CapacitySchedulerConfiguration.PREFIX - + queuePath + CapacitySchedulerConfiguration.DOT; + String keyPrefix = QueuePrefixes.getQueuePrefix(queuePath); for (Map.Entry kv : addInfo.getParams().entrySet()) { String keyValue = kv.getValue(); if (keyValue == null || keyValue.isEmpty()) { @@ -142,8 +133,7 @@ private static void addQueue( } // Unset Ordering Policy of Parent Queue converted from // Leaf Queue after addQueue - String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX - + parentPath + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY; + String queueOrderingPolicy = getOrderingPolicyConfig(parentPath); if (siblingQueues.size() == 1) { proposedConf.unset(queueOrderingPolicy); confUpdate.put(queueOrderingPolicy, null); @@ -156,9 +146,8 @@ private static void updateQueue(QueueConfigInfo updateInfo, if (updateInfo == null) { return; } - String queuePath = updateInfo.getQueue(); - String keyPrefix = CapacitySchedulerConfiguration.PREFIX - + queuePath + CapacitySchedulerConfiguration.DOT; + QueuePath queuePath = new QueuePath(updateInfo.getQueue()); + String keyPrefix = QueuePrefixes.getQueuePrefix(queuePath); for (Map.Entry kv : updateInfo.getParams().entrySet()) { String keyValue = kv.getValue(); if (keyValue == null || keyValue.isEmpty()) { @@ -171,11 +160,16 @@ private static void updateQueue(QueueConfigInfo updateInfo, } } - private static List getSiblingQueues(String queuePath, Configuration conf) { - String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.')); - String childQueuesKey = CapacitySchedulerConfiguration.PREFIX + - parentQueue + CapacitySchedulerConfiguration.DOT + - CapacitySchedulerConfiguration.QUEUES; + private static List getSiblingQueues(QueuePath queuePath, Configuration conf) { + String childQueuesKey = getQueuesConfig(queuePath.getParentObject()); return new ArrayList<>(conf.getTrimmedStringCollection(childQueuesKey)); } + + private static String getQueuesConfig(QueuePath queuePath) { + return QueuePrefixes.getQueuePrefix(queuePath) + CapacitySchedulerConfiguration.QUEUES; + } + + private static String getOrderingPolicyConfig(QueuePath queuePath) { + return QueuePrefixes.getQueuePrefix(queuePath) + CapacitySchedulerConfiguration.ORDERING_POLICY; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java index 7eb577d9c1271..0dbceba172b97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java @@ -22,23 +22,31 @@ import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; import java.util.List; public class TestQueuePath { private static final String TEST_QUEUE = "root.level_1.level_2.level_3"; + private static final QueuePath TEST_QUEUE_PATH = new QueuePath(TEST_QUEUE); + private static final QueuePath QUEUE_PATH_WITH_EMPTY_PART = new QueuePath("root..level_2"); + private static final QueuePath ROOT_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT); + private static final QueuePath EMPTY_PATH = new QueuePath(""); + private static final QueuePath ONE_LEVEL_WILDCARDED_TEST_PATH = + new QueuePath("root.level_1.level_2.*"); + private static final QueuePath TWO_LEVEL_WILDCARDED_TEST_PATH = + new QueuePath("root.level_1.*.*"); + private static final QueuePath THREE_LEVEL_WILDCARDED_TEST_PATH = + new QueuePath("root.*.*.*"); @Test public void testCreation() { - QueuePath queuePath = new QueuePath(TEST_QUEUE); - - Assert.assertEquals(TEST_QUEUE, queuePath.getFullPath()); - Assert.assertEquals("root.level_1.level_2", queuePath.getParent()); - Assert.assertEquals("level_3", queuePath.getLeafName()); + Assert.assertEquals(TEST_QUEUE, TEST_QUEUE_PATH.getFullPath()); + Assert.assertEquals("root.level_1.level_2", TEST_QUEUE_PATH.getParent()); + Assert.assertEquals("level_3", TEST_QUEUE_PATH.getLeafName()); - QueuePath rootPath = new QueuePath(CapacitySchedulerConfiguration.ROOT); - Assert.assertNull(rootPath.getParent()); + Assert.assertNull(ROOT_PATH.getParent()); - QueuePath appendedPath = queuePath.createNewLeaf("level_4"); + QueuePath appendedPath = TEST_QUEUE_PATH.createNewLeaf("level_4"); Assert.assertEquals(TEST_QUEUE + CapacitySchedulerConfiguration.DOT + "level_4", appendedPath.getFullPath()); Assert.assertEquals("root.level_1.level_2.level_3", appendedPath.getParent()); @@ -47,11 +55,8 @@ public void testCreation() { @Test public void testEmptyPart() { - QueuePath queuePathWithEmptyPart = new QueuePath("root..level_2"); - QueuePath queuePathWithoutEmptyPart = new QueuePath(TEST_QUEUE); - - Assert.assertTrue(queuePathWithEmptyPart.hasEmptyPart()); - Assert.assertFalse(queuePathWithoutEmptyPart.hasEmptyPart()); + Assert.assertTrue(QUEUE_PATH_WITH_EMPTY_PART.hasEmptyPart()); + Assert.assertFalse(TEST_QUEUE_PATH.hasEmptyPart()); } @Test @@ -66,14 +71,10 @@ public void testNullPath() { @Test public void testIterator() { - QueuePath queuePath = new QueuePath(TEST_QUEUE); - QueuePath queuePathWithEmptyPart = new QueuePath("root..level_2"); - QueuePath rootPath = new QueuePath(CapacitySchedulerConfiguration.ROOT); - - List queuePathCollection = ImmutableList.copyOf(queuePath.iterator()); + List queuePathCollection = ImmutableList.copyOf(TEST_QUEUE_PATH.iterator()); List queuePathWithEmptyPartCollection = ImmutableList.copyOf( - queuePathWithEmptyPart.iterator()); - List rootPathCollection = ImmutableList.copyOf(rootPath.iterator()); + QUEUE_PATH_WITH_EMPTY_PART.iterator()); + List rootPathCollection = ImmutableList.copyOf(ROOT_PATH.iterator()); Assert.assertEquals(4, queuePathCollection.size()); Assert.assertEquals(CapacitySchedulerConfiguration.ROOT, queuePathCollection.get(0)); @@ -90,14 +91,10 @@ public void testIterator() { @Test public void testReversePathIterator() { - QueuePath queuePath = new QueuePath(TEST_QUEUE); - QueuePath queuePathWithEmptyPart = new QueuePath("root..level_2"); - QueuePath rootPath = new QueuePath(CapacitySchedulerConfiguration.ROOT); - - List queuePathCollection = ImmutableList.copyOf(queuePath.reverseIterator()); + List queuePathCollection = ImmutableList.copyOf(TEST_QUEUE_PATH.reverseIterator()); List queuePathWithEmptyPartCollection = ImmutableList.copyOf( - queuePathWithEmptyPart.reverseIterator()); - List rootPathCollection = ImmutableList.copyOf(rootPath.reverseIterator()); + QUEUE_PATH_WITH_EMPTY_PART.reverseIterator()); + List rootPathCollection = ImmutableList.copyOf(ROOT_PATH.reverseIterator()); Assert.assertEquals(4, queuePathCollection.size()); Assert.assertEquals(CapacitySchedulerConfiguration.ROOT, @@ -126,4 +123,101 @@ public void testEquals() { Assert.assertEquals(empty, emptySame); Assert.assertNotEquals(null, queuePath); } + + @Test + public void testInvalidPath() { + Assert.assertFalse(TEST_QUEUE_PATH.isInvalid()); + Assert.assertFalse(ROOT_PATH.isInvalid()); + Assert.assertTrue(EMPTY_PATH.isInvalid()); + Assert.assertTrue(new QueuePath("invalidPath").isInvalid()); + } + + @Test + public void testGetParentObject() { + Assert.assertEquals(new QueuePath("root.level_1.level_2"), + TEST_QUEUE_PATH.getParentObject()); + Assert.assertEquals(ROOT_PATH, new QueuePath("root.level_1").getParentObject()); + Assert.assertNull(ROOT_PATH.getParentObject()); + } + + @Test + public void testGetPathComponents() { + Assert.assertArrayEquals(TEST_QUEUE_PATH.getPathComponents(), + new String[] { "root", "level_1", "level_2", "level_3" }); + Assert.assertArrayEquals(ROOT_PATH.getPathComponents(), new String[] { "root" }); + Assert.assertArrayEquals(EMPTY_PATH.getPathComponents(), new String[] { "" }); + } + + @Test + public void testWildcardedQueuePathsWithOneLevelWildCard() { + int maxAutoCreatedQueueDepth = 1; + + List expectedPaths = new ArrayList<>(); + expectedPaths.add(TEST_QUEUE_PATH); + expectedPaths.add(ONE_LEVEL_WILDCARDED_TEST_PATH); + + List wildcardedPaths = TEST_QUEUE_PATH + .getWildcardedQueuePaths(maxAutoCreatedQueueDepth); + + Assert.assertEquals(expectedPaths, wildcardedPaths); + } + + @Test + public void testWildcardedQueuePathsWithTwoLevelWildCard() { + int maxAutoCreatedQueueDepth = 2; + + List expectedPaths = new ArrayList<>(); + expectedPaths.add(TEST_QUEUE_PATH); + expectedPaths.add(ONE_LEVEL_WILDCARDED_TEST_PATH); + expectedPaths.add(TWO_LEVEL_WILDCARDED_TEST_PATH); + + List wildcardedPaths = TEST_QUEUE_PATH + .getWildcardedQueuePaths(maxAutoCreatedQueueDepth); + + Assert.assertEquals(expectedPaths, wildcardedPaths); + } + + @Test + public void testWildcardedQueuePathsWithThreeLevelWildCard() { + int maxAutoCreatedQueueDepth = 3; + + List expectedPaths = new ArrayList<>(); + expectedPaths.add(TEST_QUEUE_PATH); + expectedPaths.add(ONE_LEVEL_WILDCARDED_TEST_PATH); + expectedPaths.add(TWO_LEVEL_WILDCARDED_TEST_PATH); + expectedPaths.add(THREE_LEVEL_WILDCARDED_TEST_PATH); + + List wildcardedPaths = TEST_QUEUE_PATH + .getWildcardedQueuePaths(maxAutoCreatedQueueDepth); + + Assert.assertEquals(expectedPaths, wildcardedPaths); + } + + @Test + public void testWildcardingWhenMaxACQDepthIsGreaterThanQueuePathDepth() { + int maxAutoCreatedQueueDepth = 4; + + List expectedPaths = new ArrayList<>(); + expectedPaths.add(TEST_QUEUE_PATH); + expectedPaths.add(ONE_LEVEL_WILDCARDED_TEST_PATH); + expectedPaths.add(TWO_LEVEL_WILDCARDED_TEST_PATH); + expectedPaths.add(THREE_LEVEL_WILDCARDED_TEST_PATH); + + List wildcardedPaths = TEST_QUEUE_PATH + .getWildcardedQueuePaths(maxAutoCreatedQueueDepth); + + Assert.assertEquals(expectedPaths, wildcardedPaths); + } + + @Test + public void testWildcardedQueuePathsWithRootPath() { + int maxAutoCreatedQueueDepth = 1; + + List expectedPaths = new ArrayList<>(); + expectedPaths.add(ROOT_PATH); + + List wildcardedPaths = ROOT_PATH.getWildcardedQueuePaths(maxAutoCreatedQueueDepth); + + Assert.assertEquals(expectedPaths, wildcardedPaths); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 665c8a92b5af2..b77414b334bb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -146,9 +146,9 @@ public void testSchedulerResponseWeightModeWithAutoCreatedQueues() throws Exception { Configuration config = CSConfigGenerator .createWeightConfigWithAutoQueueCreationEnabled(); - QueuePath autoPArent1 = new QueuePath("root.autoParent1"); + QueuePath autoParent1 = new QueuePath("root.autoParent1"); config.setInt(QueuePrefixes - .getQueuePrefix(autoPArent1) + + .getQueuePrefix(autoParent1) + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + "maximum-applications", 300); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java index 82c0c3b83075c..7422334655ca9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java @@ -33,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.DEDICATED_QUEUE_PATH; import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.DEFAULT_QUEUE_PATH; import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.ROOT_QUEUE_PATH; import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED_FULL; @@ -87,7 +88,7 @@ protected void setupResourceManager() throws IOException { // Define dedicated queues String[] queues = new String[]{QUEUE_DEFAULT, QUEUE_DEDICATED}; conf.setQueues(ROOT_QUEUE_PATH, queues); - conf.setCapacity(DEFAULT_QUEUE_PATH, 80); + conf.setCapacity(DEDICATED_QUEUE_PATH, 80); conf.setReservable(QUEUE_DEDICATED_FULL, true); conf.setClass(YarnConfiguration.RM_SCHEDULER, From 72f7aa6b2edf698fe22cf705c959dc884e981e67 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Tue, 13 Jun 2023 10:20:39 +0200 Subject: [PATCH 16/21] YARN-11041. Review fixes - continued replacing Change-Id: Id58a7e82338b01d237ec2887a0258d47c32acdee --- .../AbstractReservationSystem.java | 9 ++-- .../reservation/CapacityOverTimePolicy.java | 4 +- .../ReservationSchedulerConfiguration.java | 23 +++++---- .../planning/SimpleCapacityReplanner.java | 3 +- .../scheduler/capacity/CapacityScheduler.java | 2 +- .../CapacitySchedulerConfiguration.java | 51 +++++++++---------- .../CapacitySchedulerQueueManager.java | 2 +- .../scheduler/capacity/PlanQueue.java | 2 +- .../fair/AllocationConfiguration.java | 25 ++++----- .../scheduler/fair/FairScheduler.java | 10 ++-- .../CapacityReservationsACLsManager.java | 2 +- .../security/FairReservationsACLsManager.java | 3 +- .../webapp/dao/FairSchedulerQueueInfo.java | 7 ++- .../ReservationACLsTestBase.java | 6 +-- .../ReservationSystemTestUtil.java | 15 +++--- .../planning/TestSimpleCapacityReplanner.java | 3 +- .../TestCapacitySchedulerDynamicBehavior.java | 2 +- .../fair/TestAllocationFileLoaderService.java | 25 +++++---- .../webapp/TestRMWebServicesReservation.java | 5 +- .../MockDefaultRequestInterceptorREST.java | 2 +- .../TestableFederationInterceptorREST.java | 2 +- 21 files changed, 110 insertions(+), 93 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java index d9e4be9e5225b..b0b41f9697dba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java @@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.CapacityReservationsACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.FairReservationsACLsManager; @@ -430,7 +431,7 @@ protected Plan initializePlan(String planQueueName) throws YarnException { Plan plan = new InMemoryPlan(getRootQueueMetrics(), adPolicy, getAgent(planQueuePath), totCap, planStepSize, rescCalc, minAllocation, maxAllocation, planQueueName, getReplanner(planQueuePath), - getReservationSchedulerConfiguration().getMoveOnExpiry(planQueuePath), + getReservationSchedulerConfiguration().getMoveOnExpiry(new QueuePath(planQueuePath)), maxPeriodicity, rmContext); LOG.info("Initialized plan {} based on reservable queue {}", plan.toString(), planQueueName); @@ -440,7 +441,7 @@ maxAllocation, planQueueName, getReplanner(planQueuePath), protected Planner getReplanner(String planQueueName) { ReservationSchedulerConfiguration reservationConfig = getReservationSchedulerConfiguration(); - String plannerClassName = reservationConfig.getReplanner(planQueueName); + String plannerClassName = reservationConfig.getReplanner(new QueuePath(planQueueName)); LOG.info("Using Replanner: " + plannerClassName + " for queue: " + planQueueName); try { @@ -463,7 +464,7 @@ protected Planner getReplanner(String planQueueName) { protected ReservationAgent getAgent(String queueName) { ReservationSchedulerConfiguration reservationConfig = getReservationSchedulerConfiguration(); - String agentClassName = reservationConfig.getReservationAgent(queueName); + String agentClassName = reservationConfig.getReservationAgent(new QueuePath(queueName)); LOG.info("Using Agent: " + agentClassName + " for queue: " + queueName); try { Class agentClazz = conf.getClassByName(agentClassName); @@ -487,7 +488,7 @@ protected SharingPolicy getAdmissionPolicy(String queueName) { ReservationSchedulerConfiguration reservationConfig = getReservationSchedulerConfiguration(); String admissionPolicyClassName = - reservationConfig.getReservationAdmissionPolicy(queueName); + reservationConfig.getReservationAdmissionPolicy(new QueuePath(queueName)); LOG.info("Using AdmissionPolicy: " + admissionPolicyClassName + " for queue: " + queueName); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java index 1f3f9bc4ba6bc..db9794d6f6885 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation.RLEOperator; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningQuotaException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.util.resource.Resources; import java.util.Map; @@ -57,9 +58,10 @@ public class CapacityOverTimePolicy extends NoOverCommitPolicy { private float maxAvg; @Override - public void init(String reservationQueuePath, + public void init(String reservationQueue, ReservationSchedulerConfiguration conf) { this.conf = conf; + QueuePath reservationQueuePath = new QueuePath(reservationQueue); validWindow = this.conf.getReservationWindow(reservationQueuePath); maxInst = this.conf.getInstantaneousMaxCapacity(reservationQueuePath) / 100; maxAvg = this.conf.getAverageCapacity(reservationQueuePath) / 100; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java index 740b88c40c458..d6bf1ce813363 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java @@ -23,6 +23,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ReservationACL; import org.apache.hadoop.yarn.api.records.ReservationDefinition; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import java.util.Map; @@ -69,7 +70,7 @@ public ReservationSchedulerConfiguration( * @param queue name of the queue * @return true if the queue participates in reservation based scheduling */ - public abstract boolean isReservable(String queue); + public abstract boolean isReservable(QueuePath queue); /** * Gets a map containing the {@link AccessControlList} of users for each @@ -80,7 +81,7 @@ public ReservationSchedulerConfiguration( * which contains a list of users that have the specified permission level. */ public abstract Map getReservationAcls( - String queue); + QueuePath queue); /** * Gets the length of time in milliseconds for which the {@link SharingPolicy} @@ -89,7 +90,7 @@ public abstract Map getReservationAcls( * @return length in time in milliseconds for which to check the * {@link SharingPolicy} */ - public long getReservationWindow(String queue) { + public long getReservationWindow(QueuePath queue) { return DEFAULT_RESERVATION_WINDOW; } @@ -100,7 +101,7 @@ public long getReservationWindow(String queue) { * @param queue name of the queue * @return average capacity allowed by the {@link SharingPolicy} */ - public float getAverageCapacity(String queue) { + public float getAverageCapacity(QueuePath queue) { return DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER; } @@ -109,7 +110,7 @@ public float getAverageCapacity(String queue) { * @param queue name of the queue * @return maximum allowed capacity at any time */ - public float getInstantaneousMaxCapacity(String queue) { + public float getInstantaneousMaxCapacity(QueuePath queue) { return DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER; } @@ -118,7 +119,7 @@ public float getInstantaneousMaxCapacity(String queue) { * @param queue name of the queue * @return the class name of the {@link SharingPolicy} */ - public String getReservationAdmissionPolicy(String queue) { + public String getReservationAdmissionPolicy(QueuePath queue) { return DEFAULT_RESERVATION_ADMISSION_POLICY; } @@ -128,7 +129,7 @@ public String getReservationAdmissionPolicy(String queue) { * @param queue name of the queue * @return the class name of the {@code ReservationAgent} */ - public String getReservationAgent(String queue) { + public String getReservationAgent(QueuePath queue) { return DEFAULT_RESERVATION_AGENT_NAME; } @@ -137,7 +138,7 @@ public String getReservationAgent(String queue) { * @param queuePath name of the queue * @return true if reservation queues should be visible */ - public boolean getShowReservationAsQueues(String queuePath) { + public boolean getShowReservationAsQueues(QueuePath queuePath) { return DEFAULT_SHOW_RESERVATIONS_AS_QUEUES; } @@ -147,7 +148,7 @@ public boolean getShowReservationAsQueues(String queuePath) { * @param queue name of the queue * @return the class name of the {@code Planner} */ - public String getReplanner(String queue) { + public String getReplanner(QueuePath queue) { return DEFAULT_RESERVATION_PLANNER_NAME; } @@ -158,7 +159,7 @@ public String getReplanner(String queue) { * @return true if application should be moved, false if they need to be * killed */ - public boolean getMoveOnExpiry(String queue) { + public boolean getMoveOnExpiry(QueuePath queue) { return DEFAULT_RESERVATION_MOVE_ON_EXPIRY; } @@ -168,7 +169,7 @@ public boolean getMoveOnExpiry(String queue) { * @param queue name of the queue * @return the time in milliseconds for which to check constraints */ - public long getEnforcementWindow(String queue) { + public long getEnforcementWindow(QueuePath queue) { return DEFAULT_RESERVATION_ENFORCEMENT_WINDOW; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/SimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/SimpleCapacityReplanner.java index 38880aeed524b..f14a87ed8cd1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/SimpleCapacityReplanner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/SimpleCapacityReplanner.java @@ -23,6 +23,7 @@ import java.util.Set; import java.util.TreeSet; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.api.records.ReservationDefinition; @@ -72,7 +73,7 @@ public SimpleCapacityReplanner() { @Override public void init(String planQueueName, ReservationSchedulerConfiguration conf) { - this.lengthOfCheckZone = conf.getEnforcementWindow(planQueueName); + this.lengthOfCheckZone = conf.getEnforcementWindow(new QueuePath(planQueueName)); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 2c61fe61adb69..31ef696560303 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2537,7 +2537,7 @@ private String resolveReservationQueueName(String queueName, if (queue == null) { // reservation has terminated during failover if (isRecovering && conf.getMoveOnExpiry( - getQueue(queueName).getQueuePath())) { + getQueue(queueName).getQueuePathObject())) { // move to the default child queue of the plan return getDefaultReservationQueueName(queueName); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 8e191d0fb0f70..378de3f8ca596 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1027,11 +1027,11 @@ private static Map getACLsFromProperties( } @Override - public Map getReservationAcls(String + public Map getReservationAcls(QueuePath queue) { Map resAcls = new HashMap<>(); for (ReservationACL acl : ReservationACL.values()) { - resAcls.put(acl, getReservationAcl(new QueuePath(queue), acl)); + resAcls.put(acl, getReservationAcl(queue, acl)); } return resAcls; } @@ -1543,39 +1543,38 @@ void setWorkflowPriorityMappings( .getWorkflowPriorityMappingStr(workflowPriorityMappings)); } - public boolean isReservable(String queue) { + public boolean isReservable(QueuePath queue) { boolean isReservable = - getBoolean(getQueuePrefix(new QueuePath(queue)) + IS_RESERVABLE, false); + getBoolean(getQueuePrefix(queue) + IS_RESERVABLE, false); return isReservable; } - public void setReservable(String queue, boolean isReservable) { - QueuePath queuePath = new QueuePath(queue); - setBoolean(getQueuePrefix(queuePath) + IS_RESERVABLE, isReservable); + public void setReservable(QueuePath queue, boolean isReservable) {; + setBoolean(getQueuePrefix(queue) + IS_RESERVABLE, isReservable); LOG.debug("here setReservableQueue: queuePrefix={}, isReservableQueue={}", - getQueuePrefix(queuePath), isReservable(queue)); + getQueuePrefix(queue), isReservable(queue)); } @Override - public long getReservationWindow(String queue) { + public long getReservationWindow(QueuePath queue) { long reservationWindow = - getLong(getQueuePrefix(new QueuePath(queue)) + RESERVATION_WINDOW, + getLong(getQueuePrefix(queue) + RESERVATION_WINDOW, DEFAULT_RESERVATION_WINDOW); return reservationWindow; } @Override - public float getAverageCapacity(String queue) { + public float getAverageCapacity(QueuePath queue) { float avgCapacity = - getFloat(getQueuePrefix(new QueuePath(queue)) + AVERAGE_CAPACITY, + getFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY, MAXIMUM_CAPACITY_VALUE); return avgCapacity; } @Override - public float getInstantaneousMaxCapacity(String queue) { + public float getInstantaneousMaxCapacity(QueuePath queue) { float instMaxCapacity = - getFloat(getQueuePrefix(new QueuePath(queue)) + INSTANTANEOUS_MAX_CAPACITY, + getFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY, MAXIMUM_CAPACITY_VALUE); return instMaxCapacity; } @@ -1594,9 +1593,9 @@ public void setAverageCapacity(QueuePath queue, float avgCapacity) { } @Override - public String getReservationAdmissionPolicy(String queue) { + public String getReservationAdmissionPolicy(QueuePath queue) { String reservationPolicy = - get(getQueuePrefix(new QueuePath(queue)) + RESERVATION_ADMISSION_POLICY, + get(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY, DEFAULT_RESERVATION_ADMISSION_POLICY); return reservationPolicy; } @@ -1607,9 +1606,9 @@ public void setReservationAdmissionPolicy(QueuePath queue, } @Override - public String getReservationAgent(String queue) { + public String getReservationAgent(QueuePath queue) { String reservationAgent = - get(getQueuePrefix(new QueuePath(queue)) + RESERVATION_AGENT_NAME, + get(getQueuePrefix(queue) + RESERVATION_AGENT_NAME, DEFAULT_RESERVATION_AGENT_NAME); return reservationAgent; } @@ -1619,34 +1618,34 @@ public void setReservationAgent(QueuePath queue, String reservationPolicy) { } @Override - public boolean getShowReservationAsQueues(String queuePath) { + public boolean getShowReservationAsQueues(QueuePath queuePath) { boolean showReservationAsQueues = - getBoolean(getQueuePrefix(new QueuePath(queuePath)) + getBoolean(getQueuePrefix(queuePath) + RESERVATION_SHOW_RESERVATION_AS_QUEUE, DEFAULT_SHOW_RESERVATIONS_AS_QUEUES); return showReservationAsQueues; } @Override - public String getReplanner(String queue) { + public String getReplanner(QueuePath queue) { String replanner = - get(getQueuePrefix(new QueuePath(queue)) + RESERVATION_PLANNER_NAME, + get(getQueuePrefix(queue) + RESERVATION_PLANNER_NAME, DEFAULT_RESERVATION_PLANNER_NAME); return replanner; } @Override - public boolean getMoveOnExpiry(String queue) { + public boolean getMoveOnExpiry(QueuePath queue) { boolean killOnExpiry = - getBoolean(getQueuePrefix(new QueuePath(queue)) + RESERVATION_MOVE_ON_EXPIRY, + getBoolean(getQueuePrefix(queue) + RESERVATION_MOVE_ON_EXPIRY, DEFAULT_RESERVATION_MOVE_ON_EXPIRY); return killOnExpiry; } @Override - public long getEnforcementWindow(String queue) { + public long getEnforcementWindow(QueuePath queue) { long enforcementWindow = - getLong(getQueuePrefix(new QueuePath(queue)) + RESERVATION_ENFORCEMENT_WINDOW, + getLong(getQueuePrefix(queue) + RESERVATION_ENFORCEMENT_WINDOW, DEFAULT_RESERVATION_ENFORCEMENT_WINDOW); return enforcementWindow; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 66e3ce053ad37..6e4c1a03cb27a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -241,7 +241,7 @@ static CSQueue parseQueue( List childQueueNames = conf.getQueues(fullQueuePath); CSQueue oldQueue = oldQueues.get(fullQueuePath.getFullPath()); - boolean isReservableQueue = conf.isReservable(fullQueuePath.getFullPath()); + boolean isReservableQueue = conf.isReservable(fullQueuePath); boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(fullQueuePath); // if a queue is eligible for auto queue creation v2 it must be a ParentQueue // (even if it is empty) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index e4540482d10da..85265a9054b32 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -55,7 +55,7 @@ public PlanQueue(CapacitySchedulerQueueContext queueContext, String queueName, String queuePath = super.getQueuePath(); QueuePath queuePathObject = super.getQueuePathObject(); int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePathObject); - showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath); + showReservationsAsQueues = conf.getShowReservationAsQueues(queuePathObject); if (maxAppsForReservation < 0) { maxAppsForReservation = (int) (CapacitySchedulerConfiguration. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index b3705c600575a..805ebef4316c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileParser; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.QueueProperties; import org.apache.hadoop.yarn.util.resource.Resources; @@ -194,7 +195,7 @@ public Map> getQueueAcls() { * Get the map of reservation ACLs to {@link AccessControlList} for the * specified queue. */ - public Map getReservationAcls(String + public Map getReservationAcls(QueuePath queue) { return this.resAcls.get(queue); } @@ -341,52 +342,52 @@ public Map> getConfiguredQueues() { } @Override - public boolean isReservable(String queue) { - return reservableQueues.contains(queue); + public boolean isReservable(QueuePath queue) { + return reservableQueues.contains(queue.getFullPath()); } @Override - public long getReservationWindow(String queue) { + public long getReservationWindow(QueuePath queue) { return globalReservationQueueConfig.getReservationWindowMsec(); } @Override - public float getAverageCapacity(String queue) { + public float getAverageCapacity(QueuePath queue) { return globalReservationQueueConfig.getAvgOverTimeMultiplier() * 100; } @Override - public float getInstantaneousMaxCapacity(String queue) { + public float getInstantaneousMaxCapacity(QueuePath queue) { return globalReservationQueueConfig.getMaxOverTimeMultiplier() * 100; } @Override - public String getReservationAdmissionPolicy(String queue) { + public String getReservationAdmissionPolicy(QueuePath queue) { return globalReservationQueueConfig.getReservationAdmissionPolicy(); } @Override - public String getReservationAgent(String queue) { + public String getReservationAgent(QueuePath queue) { return globalReservationQueueConfig.getReservationAgent(); } @Override - public boolean getShowReservationAsQueues(String queue) { + public boolean getShowReservationAsQueues(QueuePath queue) { return globalReservationQueueConfig.shouldShowReservationAsQueues(); } @Override - public String getReplanner(String queue) { + public String getReplanner(QueuePath queue) { return globalReservationQueueConfig.getPlanner(); } @Override - public boolean getMoveOnExpiry(String queue) { + public boolean getMoveOnExpiry(QueuePath queue) { return globalReservationQueueConfig.shouldMoveOnExpiry(); } @Override - public long getEnforcementWindow(String queue) { + public long getEnforcementWindow(QueuePath queue) { return globalReservationQueueConfig.getEnforcementWindowMsec(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 32a3b7be4be68..402baea9a8c8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -76,6 +76,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils.MaxResourceValidationResult; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -1363,7 +1364,8 @@ private String resolveReservationQueueName(String queueName, readLock.lock(); try { FSQueue queue = queueMgr.getQueue(queueName); - if ((queue == null) || !allocConf.isReservable(queue.getQueueName())) { + QueuePath queuePath = new QueuePath(queue.getQueueName()); + if ((queue == null) || !allocConf.isReservable(queuePath)) { return queueName; } // Use fully specified name from now on (including root. prefix) @@ -1373,7 +1375,7 @@ private String resolveReservationQueueName(String queueName, queue = queueMgr.getQueue(resQName); if (queue == null) { // reservation has terminated during failover - if (isRecovering && allocConf.getMoveOnExpiry(queueName)) { + if (isRecovering && allocConf.getMoveOnExpiry(queuePath)) { // move to the default child queue of the plan return getDefaultQueueForPlanQueue(queueName); } @@ -1972,7 +1974,7 @@ public Set getPlanQueues() throws YarnException { Set planQueues = new HashSet(); for (FSQueue fsQueue : queueMgr.getQueues()) { String queueName = fsQueue.getName(); - if (allocConf.isReservable(queueName)) { + if (allocConf.isReservable(new QueuePath(queueName))) { planQueues.add(queueName); } } @@ -2013,7 +2015,7 @@ public void removeQueue(String queueName) throws YarnException { private String handleMoveToPlanQueue(String targetQueueName) { FSQueue dest = queueMgr.getQueue(targetQueueName); - if (dest != null && allocConf.isReservable(dest.getQueueName())) { + if (dest != null && allocConf.isReservable(new QueuePath(dest.getQueueName()))) { // use the default child reservation queue of the plan targetQueueName = getDefaultQueueForPlanQueue(targetQueueName); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java index 531d2a315331b..418f822187fa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java @@ -39,7 +39,7 @@ public CapacityReservationsACLsManager(ResourceScheduler scheduler, for (String planQueue : scheduler.getPlanQueues()) { CSQueue queue = ((CapacityScheduler) scheduler).getQueue(planQueue); reservationAcls.put(planQueue, - csConf.getReservationAcls(queue.getQueuePath())); + csConf.getReservationAcls(queue.getQueuePathObject())); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java index 09f147f89ea14..a2d05de4e39b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; @@ -35,7 +36,7 @@ public FairReservationsACLsManager(ResourceScheduler scheduler, AllocationConfiguration aConf = ((FairScheduler) scheduler) .getAllocationConfiguration(); for (String planQueue : scheduler.getPlanQueues()) { - reservationAcls.put(planQueue, aConf.getReservationAcls(planQueue)); + reservationAcls.put(planQueue, aConf.getReservationAcls(new QueuePath(planQueue))); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java index 446b7928bf93b..65e5a700d6be8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -27,6 +27,8 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -111,8 +113,9 @@ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { reservedContainers = queue.getMetrics().getReservedContainers(); pendingContainers = queue.getMetrics().getPendingContainers(); - if (allocConf.isReservable(queueName) && - !allocConf.getShowReservationAsQueues(queueName)) { + QueuePath queuePath = new QueuePath(queueName); + if (allocConf.isReservable(queuePath) && + !allocConf.getShowReservationAsQueues(queuePath)) { return; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java index 6d5668ed010ab..a0fe708c77e20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java @@ -505,9 +505,9 @@ private static Configuration createCapacitySchedulerConfiguration() { csConf.setCapacity(A_QUEUE_PATH, 50f); csConf.setCapacity(B_QUEUE_PATH, 20f); csConf.setCapacity(C_QUEUE_PATH, 30f); - csConf.setReservable(A_QUEUE_PATH.getFullPath(), true); - csConf.setReservable(B_QUEUE_PATH.getFullPath(), true); - csConf.setReservable(C_QUEUE_PATH.getFullPath(), true); + csConf.setReservable(A_QUEUE_PATH, true); + csConf.setReservable(B_QUEUE_PATH, true); + csConf.setReservable(C_QUEUE_PATH, true); // Set up ACLs on Queue A Map reservationAclsOnQueueA = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index c8a034c545bdf..7e172aee35308 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -100,10 +100,11 @@ public static ReservationSchedulerConfiguration createConf( ReservationSchedulerConfiguration realConf = new CapacitySchedulerConfiguration(); ReservationSchedulerConfiguration conf = spy(realConf); - when(conf.getReservationWindow(reservationQ)).thenReturn(timeWindow); - when(conf.getInstantaneousMaxCapacity(reservationQ)) + QueuePath reservationQueuePath = new QueuePath(reservationQ); + when(conf.getReservationWindow(reservationQueuePath)).thenReturn(timeWindow); + when(conf.getInstantaneousMaxCapacity(reservationQueuePath)) .thenReturn(instConstraint); - when(conf.getAverageCapacity(reservationQ)).thenReturn(avgConstraint); + when(conf.getAverageCapacity(reservationQueuePath)).thenReturn(avgConstraint); return conf; } @@ -315,7 +316,7 @@ public static void setupQueueConfiguration( conf.setCapacity(A, 10); conf.setCapacity(DEDICATED, 80); // Set as reservation queue - conf.setReservable(DEDICATED.getFullPath(), true); + conf.setReservable(DEDICATED, true); // Define 2nd-level queues conf.setQueues(A, new String[] {"a1", "a2"}); @@ -330,7 +331,7 @@ public static void setupDynamicQueueConfiguration( new String[] {RESERVATION_Q_SHORT}); conf.setCapacity(DEDICATED, 100); // Set as reservation queue - conf.setReservable(DEDICATED.getFullPath(), true); + conf.setReservable(DEDICATED, true); } public static String getFullReservationQueueName() { @@ -359,12 +360,12 @@ public static void updateQueueConfiguration( conf.setCapacity(A, 5); conf.setCapacity(DEDICATED, 10); // Set as reservation queue - conf.setReservable(DEDICATED.getFullPath(), true); + conf.setReservable(DEDICATED, true); final QueuePath newQueue = new QueuePath(prefix + newQ); conf.setCapacity(newQueue, 80); // Set as reservation queue - conf.setReservable(prefix + newQ, true); + conf.setReservable(newQueue, true); // Define 2nd-level queues conf.setQueues(A, new String[] { "a1", "a2" }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java index b8521969b343c..1209c7e757ce9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.SharingPolicy; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -74,7 +75,7 @@ public void testReplanningPlanCapacityLoss() throws PlanningException { RMContext context = ReservationSystemTestUtil.createMockRMContext(); ReservationSchedulerConfiguration conf = mock(ReservationSchedulerConfiguration.class); - when(conf.getEnforcementWindow(any(String.class))).thenReturn(6L); + when(conf.getEnforcementWindow(any(QueuePath.class))).thenReturn(6L); enf.init("blah", conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java index afd49aae58163..c753377347955 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java @@ -304,7 +304,7 @@ private void setupPlanQueueConfiguration(CapacitySchedulerConfiguration conf) { conf.setCapacity(B3, B3_CAPACITY); conf.setUserLimitFactor(B3, 100.0f); - conf.setReservable(A.getFullPath(), true); + conf.setReservable(A, true); conf.setReservationWindow(A, 86400 * 1000); conf.setAverageCapacity(A, 1.0f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 9fb76cf261ea3..26ddb3aabe195 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementRule; @@ -889,8 +890,10 @@ public void testReservableQueue() throws Exception { AllocationConfiguration allocConf = confHolder.allocConf; String reservableQueueName = "root.reservable"; String nonreservableQueueName = "root.other"; - assertFalse(allocConf.isReservable(nonreservableQueueName)); - assertTrue(allocConf.isReservable(reservableQueueName)); + QueuePath nonreservableQueuePath = new QueuePath(nonreservableQueueName); + QueuePath reservableQueuePath = new QueuePath(reservableQueueName); + assertFalse(allocConf.isReservable(nonreservableQueuePath)); + assertTrue(allocConf.isReservable(reservableQueuePath)); Map> configuredQueues = allocConf.getConfiguredQueues(); assertTrue("reservable queue is expected be to a parent queue", @@ -899,23 +902,23 @@ public void testReservableQueue() throws Exception { configuredQueues.get(FSQueueType.LEAF) .contains(reservableQueueName)); - assertTrue(allocConf.getMoveOnExpiry(reservableQueueName)); + assertTrue(allocConf.getMoveOnExpiry(reservableQueuePath)); assertEquals(ReservationSchedulerConfiguration.DEFAULT_RESERVATION_WINDOW, - allocConf.getReservationWindow(reservableQueueName)); + allocConf.getReservationWindow(reservableQueuePath)); assertEquals(100, - allocConf.getInstantaneousMaxCapacity(reservableQueueName), 0.0001); + allocConf.getInstantaneousMaxCapacity(reservableQueuePath), 0.0001); assertEquals("DummyAgentName", - allocConf.getReservationAgent(reservableQueueName)); - assertEquals(100, allocConf.getAverageCapacity(reservableQueueName), 0.001); - assertFalse(allocConf.getShowReservationAsQueues(reservableQueueName)); + allocConf.getReservationAgent(reservableQueuePath)); + assertEquals(100, allocConf.getAverageCapacity(reservableQueuePath), 0.001); + assertFalse(allocConf.getShowReservationAsQueues(reservableQueuePath)); assertEquals("AnyAdmissionPolicy", - allocConf.getReservationAdmissionPolicy(reservableQueueName)); + allocConf.getReservationAdmissionPolicy(reservableQueuePath)); assertEquals(ReservationSchedulerConfiguration .DEFAULT_RESERVATION_PLANNER_NAME, - allocConf.getReplanner(reservableQueueName)); + allocConf.getReplanner(reservableQueuePath)); assertEquals(ReservationSchedulerConfiguration .DEFAULT_RESERVATION_ENFORCEMENT_WINDOW, - allocConf.getEnforcementWindow(reservableQueueName)); + allocConf.getEnforcementWindow(reservableQueuePath)); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java index 9989c4a377e73..13489dd9a0b8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java @@ -174,10 +174,11 @@ public void configureScheduler() { CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(conf); String[] queues = { "default", "dedicated" }; + QueuePath dedicatedQueuePath = new QueuePath("root.dedicated"); csconf.setQueues(new QueuePath("root"), queues); csconf.setCapacity(new QueuePath("root.default"), 50.0f); - csconf.setCapacity(new QueuePath("root.dedicated"), 50.0f); - csconf.setReservable("root.dedicated", true); + csconf.setCapacity(dedicatedQueuePath, 50.0f); + csconf.setReservable(dedicatedQueuePath, true); conf = csconf; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java index 395a53b8b5fe5..550547a9c7b1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java @@ -1112,7 +1112,7 @@ private MockRM setupResourceManager() throws Exception { conf.setQueues(ROOT_QUEUE_PATH, new String[] {QUEUE_DEFAULT, QUEUE_DEDICATED}); conf.setCapacity(DEDICATED_QUEUE_PATH, 80); - conf.setReservable(QUEUE_DEDICATED_FULL, true); + conf.setReservable(DEDICATED_QUEUE_PATH, true); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java index 7422334655ca9..3f539605b2897 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java @@ -89,7 +89,7 @@ protected void setupResourceManager() throws IOException { String[] queues = new String[]{QUEUE_DEFAULT, QUEUE_DEDICATED}; conf.setQueues(ROOT_QUEUE_PATH, queues); conf.setCapacity(DEDICATED_QUEUE_PATH, 80); - conf.setReservable(QUEUE_DEDICATED_FULL, true); + conf.setReservable(DEDICATED_QUEUE_PATH, true); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); From 035350e9884cb2a0d78b03a9ded896213876a946 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Wed, 14 Jun 2023 12:45:43 +0200 Subject: [PATCH 17/21] YARN-11041. Fixed SpotBugs and unit test issues Change-Id: I155d6ce1e91dafa4d992e56ffc4d2ed38b961891 --- .../capacity/CapacitySchedulerConfiguration.java | 2 +- .../resourcemanager/scheduler/capacity/PlanQueue.java | 11 +++++------ .../capacity/conf/ConfigurationUpdateAssembler.java | 6 +++--- .../scheduler/fair/AllocationConfiguration.java | 2 +- .../resourcemanager/scheduler/fair/FairScheduler.java | 5 ++++- .../scheduler/capacity/TestQueuePath.java | 8 ++++---- .../webapp/TestableFederationInterceptorREST.java | 1 - 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 378de3f8ca596..d55120fffe45b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -1549,7 +1549,7 @@ public boolean isReservable(QueuePath queue) { return isReservable; } - public void setReservable(QueuePath queue, boolean isReservable) {; + public void setReservable(QueuePath queue, boolean isReservable) { setBoolean(getQueuePrefix(queue) + IS_RESERVABLE, isReservable); LOG.debug("here setReservableQueue: queuePrefix={}, isReservableQueue={}", getQueuePrefix(queue), isReservable(queue)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java index 85265a9054b32..847c90e54589a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java @@ -52,18 +52,17 @@ public PlanQueue(CapacitySchedulerQueueContext queueContext, String queueName, // Set the reservation queue attributes for the Plan CapacitySchedulerConfiguration conf = queueContext.getConfiguration(); - String queuePath = super.getQueuePath(); - QueuePath queuePathObject = super.getQueuePathObject(); - int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePathObject); - showReservationsAsQueues = conf.getShowReservationAsQueues(queuePathObject); + QueuePath queuePath = super.getQueuePathObject(); + int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePath); + showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath); if (maxAppsForReservation < 0) { maxAppsForReservation = (int) (CapacitySchedulerConfiguration. DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * super .getAbsoluteCapacity()); } - float configuredUserLimit = conf.getUserLimit(queuePathObject); - float configuredUserLimitFactor = conf.getUserLimitFactor(queuePathObject); + float configuredUserLimit = conf.getUserLimit(queuePath); + float configuredUserLimitFactor = conf.getUserLimitFactor(queuePath); int configuredMaxAppsPerUserForReservation = (int) (maxAppsForReservation * (configuredUserLimit / 100.0f) * configuredUserLimitFactor); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java index 7a0508d034923..6213edb6a76f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java @@ -75,7 +75,7 @@ private static void removeQueue( throw new IOException("Queue " + queuePath.getFullPath() + " not found"); } siblingQueues.remove(queueName); - + QueuePath parentPath = queuePath.getParentObject(); proposedConf.setQueues(parentPath, siblingQueues.toArray( new String[0])); @@ -112,7 +112,7 @@ private static void addQueue( queueName)) { throw new IOException("Can't add existing queue " + queuePath); } - + QueuePath parentPath = queuePath.getParentObject(); List siblingQueues = proposedConf.getQueues(parentPath); siblingQueues.add(queueName); @@ -164,7 +164,7 @@ private static List getSiblingQueues(QueuePath queuePath, Configuration String childQueuesKey = getQueuesConfig(queuePath.getParentObject()); return new ArrayList<>(conf.getTrimmedStringCollection(childQueuesKey)); } - + private static String getQueuesConfig(QueuePath queuePath) { return QueuePrefixes.getQueuePrefix(queuePath) + CapacitySchedulerConfiguration.QUEUES; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 805ebef4316c5..870f7b1d7a1a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -197,7 +197,7 @@ public Map> getQueueAcls() { */ public Map getReservationAcls(QueuePath queue) { - return this.resAcls.get(queue); + return this.resAcls.get(queue.getFullPath()); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 402baea9a8c8b..dc4e842518c6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -1364,8 +1364,11 @@ private String resolveReservationQueueName(String queueName, readLock.lock(); try { FSQueue queue = queueMgr.getQueue(queueName); + if (queue == null) { + return queueName; + } QueuePath queuePath = new QueuePath(queue.getQueueName()); - if ((queue == null) || !allocConf.isReservable(queuePath)) { + if (!allocConf.isReservable(queuePath)) { return queueName; } // Use fully specified name from now on (including root. prefix) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java index 0dbceba172b97..430e1e0ee3396 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java @@ -143,9 +143,9 @@ public void testGetParentObject() { @Test public void testGetPathComponents() { Assert.assertArrayEquals(TEST_QUEUE_PATH.getPathComponents(), - new String[] { "root", "level_1", "level_2", "level_3" }); - Assert.assertArrayEquals(ROOT_PATH.getPathComponents(), new String[] { "root" }); - Assert.assertArrayEquals(EMPTY_PATH.getPathComponents(), new String[] { "" }); + new String[] {"root", "level_1", "level_2", "level_3"}); + Assert.assertArrayEquals(ROOT_PATH.getPathComponents(), new String[] {"root"}); + Assert.assertArrayEquals(EMPTY_PATH.getPathComponents(), new String[] {""}); } @Test @@ -176,7 +176,7 @@ public void testWildcardedQueuePathsWithTwoLevelWildCard() { Assert.assertEquals(expectedPaths, wildcardedPaths); } - + @Test public void testWildcardedQueuePathsWithThreeLevelWildCard() { int maxAutoCreatedQueueDepth = 3; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java index 3f539605b2897..7a9dc1b85727d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestableFederationInterceptorREST.java @@ -36,7 +36,6 @@ import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.DEDICATED_QUEUE_PATH; import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.DEFAULT_QUEUE_PATH; import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.ROOT_QUEUE_PATH; -import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED_FULL; import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEFAULT; import static org.apache.hadoop.yarn.server.router.webapp.BaseRouterWebServicesTest.QUEUE_DEDICATED; From 88cce3dc7b82b24bccf00da89ea1f682e51f3363 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Thu, 15 Jun 2023 10:06:57 +0200 Subject: [PATCH 18/21] YARN-11041. Review fixes - added unit tests for AutoCreatedQueueTemplate class Change-Id: Id5b10d9c687d90494ebcb70f2b6292de8867630d --- .../yarn/client/cli/TestSchedConfCLI.java | 2 +- .../TestAutoCreatedQueueTemplate.java | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java index 6cae066dc1e26..830fb8d5669fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.client.cli; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.junit.Before; import org.junit.Test; @@ -46,6 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index 4e81cd5c20259..5b3acf11c4fdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -183,6 +183,48 @@ public void testQueueSpecificTemplates() { } + @Test + public void testWildcardTemplateWithLimitedAutoCreatedQueueDepth() { + conf.set(getTemplateKey(TEST_QUEUE_ROOT_WILDCARD, "capacity"), "6w"); + conf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "5w"); + conf.setMaximumAutoCreatedQueueDepth(TEST_QUEUE_A, 1); + conf.setMaximumAutoCreatedQueueDepth(TEST_QUEUE_AB, 1); + + new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A) + .setTemplateEntriesForChild(conf, TEST_QUEUE_AB); + new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB) + .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); + + Assert.assertEquals("weight is not set", 6f, + conf.getNonLabeledQueueWeight(TEST_QUEUE_AB), 10e-6); + Assert.assertEquals("weight is not set", 5f, + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); + } + + @Test + public void testIgnoredTemplateWithLimitedAutoCreatedQueueDepth() { + conf.set(getTemplateKey(TEST_QUEUE_TWO_LEVEL_WILDCARDS, "capacity"), "5w"); + conf.setMaximumAutoCreatedQueueDepth(TEST_QUEUE_AB, 1); + + new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB) + .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC); + + Assert.assertEquals("weight is set incorrectly", -1f, + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6); + } + + @Test + public void testIgnoredTemplateWhenQueuePathIsInvalid() { + QueuePath invalidPath = new QueuePath("a"); + conf.set(getTemplateKey(invalidPath, "capacity"), "6w"); + AutoCreatedQueueTemplate template = + new AutoCreatedQueueTemplate(conf, invalidPath); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_AB); + + Assert.assertEquals("weight is set using invalid queue path", -1f, + conf.getNonLabeledQueueWeight(TEST_QUEUE_AB), 10e-6); + } + private String getTemplateKey(QueuePath queuePath, String entryKey) { return QueuePrefixes.getQueuePrefix(queuePath) + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey; From 37dd8ff239d5d87b1d9c228f384ad76aceee31e4 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Wed, 16 Aug 2023 15:33:25 +0200 Subject: [PATCH 19/21] YARN-11041. Aligning the changes to trunk Change-Id: Ic5acdae90659b4addc3ef23601d1528b4784b9b6 --- .../policygenerator/TestPolicyGenerator.java | 17 +- .../capacity/ManagedParentQueue.java | 8 +- .../FSConfigToCSConfigConverter.java | 5 +- .../fair/converter/FSQueueConverter.java | 27 ++-- .../WeightToPercentConverter.java | 5 +- .../WeightToWeightConverter.java | 8 +- .../resourcemanager/TestAppManager.java | 8 +- .../TestWorkPreservingRMRestart.java | 31 ++-- .../TestAutoCreatedQueueDeletionPolicy.java | 23 +-- ...CapacitySchedulerAutoCreatedQueueBase.java | 3 +- ...wQueueAutoCreationWithCapacityVectors.java | 38 ++--- .../scheduler/capacity/TestLeafQueue.java | 4 + .../scheduler/capacity/TestQueueParsing.java | 10 +- .../TestFSConfigToCSConfigConverter.java | 53 ++++--- .../fair/converter/TestFSQueueConverter.java | 147 +++++++++--------- .../TestWeightToPercentageConverter.java | 34 ++-- .../TestWeightToWeightConverter.java | 24 +-- 17 files changed, 238 insertions(+), 207 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java index 9893e85e56f29..0c6f26ab89c9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; @@ -248,13 +249,13 @@ public void testCallRM() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - final String a = CapacitySchedulerConfiguration.ROOT + ".a"; - final String b = CapacitySchedulerConfiguration.ROOT + ".b"; - final String a1 = a + ".a1"; - final String a2 = a + ".a2"; - final String b1 = b + ".b1"; - final String b2 = b + ".b2"; - final String b3 = b + ".b3"; + final QueuePath a = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".a"); + final QueuePath b = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".b"); + final QueuePath a1 = new QueuePath(a + ".a1"); + final QueuePath a2 = new QueuePath(a + ".a2"); + final QueuePath b1 = new QueuePath(b + ".b1"); + final QueuePath b2 = new QueuePath(b + ".b2"); + final QueuePath b3 = new QueuePath(b + ".b3"); float aCapacity = 10.5f; float bCapacity = 89.5f; float a1Capacity = 30; @@ -264,7 +265,7 @@ public void testCallRM() { float b3Capacity = 20; // Define top-level queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + csConf.setQueues(new QueuePath(CapacitySchedulerConfiguration.ROOT), new String[] {"a", "b"}); csConf.setCapacity(a, aCapacity); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index e0c675eba4e10..0b9079990f84c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -354,14 +354,14 @@ private void setLeafQueuesCapacityVector(AutoCreatedLeafQueue leafQueue) { .getLabelsByQueue(queuePath.getFullPath()); for (String label : templateConfiguredNodeLabels) { final String leafConfigPath = - CapacitySchedulerConfiguration.getNodeLabelPrefix( - getQueuePath() + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX, label); + QueuePrefixes.getNodeLabelPrefix( + QueuePrefixes.getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()), label); String capacityString = leafConfig.get(leafConfigPath + CAPACITY, "0"); leafQueue.setConfiguredMinCapacityVector(label, - getQueueCapacityConfigParser().parse(capacityString, leafQueue.getQueuePath())); + getQueueCapacityConfigParser().parse(capacityString, leafQueue.getQueuePathObject())); String maxCapacityString = leafConfig.get(leafConfigPath + MAXIMUM_CAPACITY, "100"); leafQueue.setConfiguredMaxCapacityVector(label, - getQueueCapacityConfigParser().parse(maxCapacityString, leafQueue.getQueuePath())); + getQueueCapacityConfigParser().parse(maxCapacityString, leafQueue.getQueuePathObject())); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index 6c8fed8182b80..ff47b606c7403 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.MappingRulesDescription; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; @@ -422,13 +423,13 @@ private void generateQueueAcl(String queue, if (!submitAcls.getGroups().isEmpty() || !submitAcls.getUsers().isEmpty() || submitAcls.isAllAllowed()) { - capacitySchedulerConfig.setAcl(queue, QueueACL.SUBMIT_APPLICATIONS, + capacitySchedulerConfig.setAcl(new QueuePath(queue), QueueACL.SUBMIT_APPLICATIONS, submitAcls.getAclString()); } if (!adminAcls.getGroups().isEmpty() || !adminAcls.getUsers().isEmpty() || adminAcls.isAllAllowed()) { - capacitySchedulerConfig.setAcl(queue, QueueACL.ADMINISTER_QUEUE, + capacitySchedulerConfig.setAcl(new QueuePath(queue), QueueACL.ADMINISTER_QUEUE, adminAcls.getAclString()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index 1bbf056c148d0..5eacb34fa450b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -104,7 +105,7 @@ private void emitChildQueues(String queueName, List children) { List childQueues = children.stream() .map(child -> getQueueShortName(child.getName())) .collect(Collectors.toList()); - capacitySchedulerConfig.setQueues(queueName, childQueues.toArray(new String[0])); + capacitySchedulerConfig.setQueues(new QueuePath(queueName), childQueues.toArray(new String[0])); } } @@ -122,13 +123,13 @@ private void emitMaxAMShare(String queueName, FSQueue queue) { && queueMaxAmShare != queueMaxAMShareDefault && queueMaxAmShare != QUEUE_MAX_AM_SHARE_DISABLED) { capacitySchedulerConfig.setMaximumApplicationMasterResourcePerQueuePercent( - queueName, queueMaxAmShare); + new QueuePath(queueName), queueMaxAmShare); } if (queueMaxAmShare == QUEUE_MAX_AM_SHARE_DISABLED && queueMaxAmShare != queueMaxAMShareDefault) { capacitySchedulerConfig.setMaximumApplicationMasterResourcePerQueuePercent( - queueName, 1.0f); + new QueuePath(queueName), 1.0f); } } @@ -141,7 +142,7 @@ private void emitMaxAMShare(String queueName, FSQueue queue) { private void emitMaxParallelApps(String queueName, FSQueue queue) { if (queue.getMaxRunningApps() != MAX_RUNNING_APPS_UNSET && queue.getMaxRunningApps() != queueMaxAppsDefault) { - capacitySchedulerConfig.setMaxParallelAppsForQueue(queueName, + capacitySchedulerConfig.setMaxParallelAppsForQueue(new QueuePath(queueName), String.valueOf(queue.getMaxRunningApps())); } } @@ -161,7 +162,7 @@ private void emitMaximumCapacity(String queueName, FSQueue queue) { ruleHandler.handleMaxResources(); } - capacitySchedulerConfig.setMaximumCapacity(queueName, + capacitySchedulerConfig.setMaximumCapacity(new QueuePath(queueName), 100.0f); } @@ -194,10 +195,10 @@ private void emitMaxAllocations(String queueName, FSQueue queue) { // only emit max allocation if it differs from the parent's setting if (maxVcores != parentMaxVcores || maxMemory != parentMaxMemory) { capacitySchedulerConfig.setQueueMaximumAllocationMb( - queueName, (int) maxMemory); + new QueuePath(queueName), (int) maxMemory); capacitySchedulerConfig.setQueueMaximumAllocationVcores( - queueName, maxVcores); + new QueuePath(queueName), maxVcores); } } } @@ -210,14 +211,14 @@ private void emitMaxAllocations(String queueName, FSQueue queue) { */ private void emitPreemptionDisabled(String queueName, FSQueue queue) { if (preemptionEnabled && !queue.isPreemptable()) { - capacitySchedulerConfig.setPreemptionDisabled(queueName, true); + capacitySchedulerConfig.setPreemptionDisabled(new QueuePath(queueName), true); } } public void emitDefaultUserLimitFactor(String queueName, List children) { if (children.isEmpty() && - !capacitySchedulerConfig.isAutoQueueCreationV2Enabled(queueName)) { - capacitySchedulerConfig.setUserLimitFactor(queueName, -1.0f); + !capacitySchedulerConfig.isAutoQueueCreationV2Enabled(new QueuePath(queueName))) { + capacitySchedulerConfig.setUserLimitFactor(new QueuePath(queueName), -1.0f); } } @@ -246,16 +247,16 @@ private void emitOrderingPolicy(String queueName, FSQueue queue) { switch (policy) { case DominantResourceFairnessPolicy.NAME: - capacitySchedulerConfig.setOrderingPolicy(queueName, FAIR_POLICY); + capacitySchedulerConfig.setOrderingPolicy(new QueuePath(queueName), FAIR_POLICY); break; case FairSharePolicy.NAME: - capacitySchedulerConfig.setOrderingPolicy(queueName, FAIR_POLICY); + capacitySchedulerConfig.setOrderingPolicy(new QueuePath(queueName), FAIR_POLICY); if (drfUsed) { ruleHandler.handleFairAsDrf(queueName); } break; case FifoPolicy.NAME: - capacitySchedulerConfig.setOrderingPolicy(queueName, FIFO_POLICY); + capacitySchedulerConfig.setOrderingPolicy(new QueuePath(queueName), FIFO_POLICY); break; default: String msg = String.format("Unexpected ordering policy " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java index 600b92626db9a..0e437c112e074 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java @@ -29,6 +29,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; public class WeightToPercentConverter @@ -50,11 +51,11 @@ public void convertWeightsForChildQueues(FSQueue queue, boolean shouldAllowZeroSumCapacity = result.getRight(); capacities - .forEach((key, value) -> csConfig.setCapacity(key, value.toString())); + .forEach((key, value) -> csConfig.setCapacity(new QueuePath(key), value.toString())); if (shouldAllowZeroSumCapacity) { String queueName = queue.getName(); - csConfig.setAllowZeroCapacitySum(queueName, true); + csConfig.setAllowZeroCapacitySum(new QueuePath(queueName), true); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java index 75554fc4d2e6d..bd13e7a9a7115 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -34,13 +35,14 @@ public void convertWeightsForChildQueues(FSQueue queue, List children = queue.getChildQueues(); if (queue instanceof FSParentQueue || !children.isEmpty()) { + QueuePath queuePath = new QueuePath(queue.getName()); if (queue.getName().equals(ROOT_QUEUE)) { - csConfig.setNonLabeledQueueWeight(queue.getName(), queue.getWeight()); + csConfig.setNonLabeledQueueWeight(queuePath, queue.getWeight()); } children.forEach(fsQueue -> csConfig.setNonLabeledQueueWeight( - fsQueue.getName(), fsQueue.getWeight())); - csConfig.setAutoQueueCreationV2Enabled(queue.getName(), true); + new QueuePath(fsQueue.getName()), fsQueue.getWeight())); + csConfig.setAutoQueueCreationV2Enabled(queuePath, true); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 013caabc3a671..50fc8fd67deac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -369,11 +369,11 @@ public void testQueueSubmitWithLeafQueueName() CapacitySchedulerConfiguration(conf, false); csConf.set(PREFIX + "root.queues", "default,test"); - csConf.setCapacity("root.default", 50.0f); - csConf.setMaximumCapacity("root.default", 100.0f); + csConf.setCapacity(DEFAULT, 50.0f); + csConf.setMaximumCapacity(DEFAULT, 100.0f); - csConf.setCapacity("root.test", 50.0f); - csConf.setMaximumCapacity("root.test", 100.0f); + csConf.setCapacity(TEST, 50.0f); + csConf.setMaximumCapacity(TEST, 100.0f); MockRM newMockRM = new MockRM(csConf); RMContext newMockRMContext = newMockRM.getRMContext(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 661855c449324..e91205d60bd13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -1672,24 +1672,19 @@ public void testQueueRecoveryOnRMWorkPreservingRestart() throws Exception { return; } CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); - - csConf.setQueues( - CapacitySchedulerConfiguration.ROOT, new String[] {"default", "joe", "john"}); - csConf.setCapacity( - CapacitySchedulerConfiguration.ROOT + "." + "joe", 25); - csConf.setCapacity( - CapacitySchedulerConfiguration.ROOT + "." + "john", 25); - csConf.setCapacity( - CapacitySchedulerConfiguration.ROOT + "." + "default", 50); - - final String q1 = CapacitySchedulerConfiguration.ROOT + "." + "joe"; - final String q2 = CapacitySchedulerConfiguration.ROOT + "." + "john"; - csConf.setQueues(q1, new String[] {"test"}); - csConf.setQueues(q2, new String[] {"test"}); - csConf.setCapacity( - CapacitySchedulerConfiguration.ROOT + "." + "joe.test", 100); - csConf.setCapacity( - CapacitySchedulerConfiguration.ROOT + "." + "john.test", 100); + final QueuePath defaultPath = new QueuePath(ROOT + "." + "default"); + final QueuePath joe = new QueuePath(ROOT + "." + "joe"); + final QueuePath john = new QueuePath(ROOT + "." + "john"); + + csConf.setQueues(ROOT, new String[] {"default", "joe", "john"}); + csConf.setCapacity(joe, 25); + csConf.setCapacity(john, 25); + csConf.setCapacity(defaultPath, 50); + + csConf.setQueues(joe, new String[] {"test"}); + csConf.setQueues(john, new String[] {"test"}); + csConf.setCapacity(new QueuePath(joe.getFullPath(), "test"), 100); + csConf.setCapacity(new QueuePath(john.getFullPath(), "test"), 100); csConf.set(CapacitySchedulerConfiguration.MAPPING_RULE_JSON, "{\"rules\" : [{\"type\": \"user\", \"policy\" : \"specified\", " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueDeletionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueDeletionPolicy.java index 502b6a40ef0ac..6ed76b6341c82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueDeletionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueDeletionPolicy.java @@ -48,6 +48,11 @@ public class TestAutoCreatedQueueDeletionPolicy private CapacitySchedulerQueueManager autoQueueHandler; + public static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + public static final QueuePath ROOT_A = new QueuePath("root", "a"); + public static final QueuePath ROOT_A_A1 = QueuePath.createFromQueues("root", "a", "a1"); + public static final QueuePath ROOT_B = new QueuePath("root", "b"); + /* Create the following structure: root @@ -63,15 +68,15 @@ public void setUp() throws Exception { ResourceScheduler.class); // By default, set 3 queues, a/b, and a.a1 - csConf.setQueues("root", new String[]{"a", "b"}); - csConf.setNonLabeledQueueWeight("root", 1f); - csConf.setNonLabeledQueueWeight("root.a", 1f); - csConf.setNonLabeledQueueWeight("root.b", 1f); - csConf.setQueues("root.a", new String[]{"a1"}); - csConf.setNonLabeledQueueWeight("root.a.a1", 1f); - csConf.setAutoQueueCreationV2Enabled("root", true); - csConf.setAutoQueueCreationV2Enabled("root.a", true); - csConf.setAutoQueueCreationV2Enabled(PARENT_QUEUE, true); + csConf.setQueues(ROOT, new String[]{"a", "b"}); + csConf.setNonLabeledQueueWeight(ROOT, 1f); + csConf.setNonLabeledQueueWeight(ROOT_A, 1f); + csConf.setNonLabeledQueueWeight(ROOT_B, 1f); + csConf.setQueues(ROOT_A, new String[]{"a1"}); + csConf.setNonLabeledQueueWeight(ROOT_A_A1, 1f); + csConf.setAutoQueueCreationV2Enabled(ROOT, true); + csConf.setAutoQueueCreationV2Enabled(ROOT_A, true); + csConf.setAutoQueueCreationV2Enabled(PARENT_QUEUE_PATH, true); // Test for auto deletion when expired csConf.setAutoExpiredDeletionTime(1); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java index eb21bd472fd50..e03e59d2512c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java @@ -178,6 +178,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase { public static final String USER2 = USER + 2; public static final String USER3 = USER + 3; public static final String PARENT_QUEUE = "c"; + public static final QueuePath PARENT_QUEUE_PATH = new QueuePath(PARENT_QUEUE); public static final Set accessibleNodeLabelsOnC = new HashSet<>(); @@ -512,7 +513,7 @@ public static void setupQueueConfigurationForSingleFlexibleAutoCreatedLeafQueue( //setup new queues with one of them auto enabled // Define top-level queues // Set childQueue for root - conf.setQueues(CapacitySchedulerConfiguration.ROOT, + conf.setQueues(ROOT, new String[] {"c"}); conf.setCapacity(C, 100f); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors.java index ccbea250f5f13..d7083b3ba0c35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors.java @@ -90,14 +90,14 @@ private void createPercentageConfig() { // root // - a 25% // a and root has AQCv2 enabled - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a"}); + csConf.setQueues(ROOT, new String[]{"a"}); csConf.setCapacity(A, 25f); - csConf.setAutoQueueCreationV2Enabled(CapacitySchedulerConfiguration.ROOT, true); + csConf.setAutoQueueCreationV2Enabled(ROOT, true); csConf.setAutoQueueCreationV2Enabled(A, true); // Set up dynamic queue templates - csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"), "6.25"); + csConf.set(getTemplateKey(ROOT, "capacity"), "6.25"); csConf.set(getLeafTemplateKey(A, "capacity"), "[memory=25%, vcores=50%]"); } @@ -105,14 +105,14 @@ private void createAbsoluteConfig() { // root // - a [memory=8000, vcores=8] // a and root has AQCv2 enabled - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a"}); + csConf.setQueues(ROOT, new String[]{"a"}); csConf.setCapacity(A, "[memory=8000, vcores=8]"); - csConf.setAutoQueueCreationV2Enabled(CapacitySchedulerConfiguration.ROOT, true); + csConf.setAutoQueueCreationV2Enabled(ROOT, true); csConf.setAutoQueueCreationV2Enabled(A, true); // Set up dynamic queue templates - csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"), + csConf.set(getTemplateKey(ROOT, "capacity"), "[memory=2000mb, vcores=2]"); csConf.set(getLeafTemplateKey(A, "capacity"), "[memory=2000, vcores=4]"); @@ -131,23 +131,23 @@ private void createMixedConfig() { // - e1-auto [memory=2048, vcores=2] // Set up static queues - csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b", "d"}); + csConf.setQueues(ROOT, new String[]{"a", "b", "d"}); csConf.setCapacityVector(A, NL, "[memory=10%, vcores=2]"); csConf.setCapacityVector(B, NL, "[memory=2000, vcores=10%]"); csConf.setCapacityVector(D, NL, "[memory=10%, vcores=10%]"); csConf.setQueues(A, new String[]{"a1"}); csConf.setCapacityVector(A1, NL, "[memory=100%, vcores=100%]"); - csConf.setAutoQueueCreationV2Enabled(CapacitySchedulerConfiguration.ROOT, true); + csConf.setAutoQueueCreationV2Enabled(ROOT, true); csConf.setAutoQueueCreationV2Enabled(A, true); csConf.setAutoQueueCreationV2Enabled(D, true); // Set up dynamic queue templates - csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"), + csConf.set(getTemplateKey(ROOT, "capacity"), "[memory=2w, vcores=5w]"); - csConf.set(getParentTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"), + csConf.set(getParentTemplateKey(ROOT, "capacity"), "[memory=2w, vcores=10]"); - csConf.set(getLeafTemplateKey(CapacitySchedulerConfiguration.ROOT + ".*", "capacity"), - "[memory=2000, vcores=2]"); + csConf.set(getLeafTemplateKey(new QueuePath(CapacitySchedulerConfiguration.ROOT + ".*"), + "capacity"), "[memory=2000, vcores=2]"); csConf.set(getLeafTemplateKey(D, "capacity"), "[memory=1000, vcores=1]"); } @@ -185,7 +185,7 @@ public void testBasicAbsoluteConfiguration() throws Exception { } private void validateBasicConfiguration() throws Exception { - CSQueue a = cs.getQueue(A); + CSQueue a = cs.getQueue(A_PATH); Assert.assertEquals(8 / 32f, a.getAbsoluteCapacity(), EPSILON); Assert.assertEquals(-1f, a.getQueueCapacities().getWeight(), EPSILON); Assert.assertEquals(8000, @@ -316,18 +316,18 @@ protected AbstractLeafQueue createQueue(String queuePath) throws YarnException, return autoQueueHandler.createQueue(new QueuePath(queuePath)); } - private String getTemplateKey(String queuePath, String entryKey) { - return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + private String getTemplateKey(QueuePath queuePath, String entryKey) { + return QueuePrefixes.getQueuePrefix(queuePath) + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey; } - private String getParentTemplateKey(String queuePath, String entryKey) { - return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + private String getParentTemplateKey(QueuePath queuePath, String entryKey) { + return QueuePrefixes.getQueuePrefix(queuePath) + AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + entryKey; } - private String getLeafTemplateKey(String queuePath, String entryKey) { - return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + private String getLeafTemplateKey(QueuePath queuePath, String entryKey) { + return QueuePrefixes.getQueuePrefix(queuePath) + AutoCreatedQueueTemplate.AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + entryKey; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 730983f0abb2e..870d61539b02b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -22,6 +22,10 @@ .NO_LABEL; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler .capacity.CSQueueUtils.EPSILON; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler + .capacity.CapacitySchedulerConfiguration.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler + .capacity.CapacitySchedulerConfiguration.ROOT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 18120a75a0c10..f5099aa5bd692 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -167,8 +167,7 @@ private void setupQueueConfigurationWithSpacesShouldBeTrimmed( CapacitySchedulerConfiguration conf) { // Define top-level queues conf.set( - CapacitySchedulerConfiguration - .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) + QueuePrefixes.getQueuePrefix(ROOT) + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); conf.setCapacity(A, 10); @@ -182,8 +181,7 @@ private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed( CapacitySchedulerConfiguration conf) { // Define top-level queues conf.set( - CapacitySchedulerConfiguration - .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) + QueuePrefixes.getQueuePrefix(ROOT) + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); conf.setCapacity(A, 10); @@ -193,7 +191,7 @@ private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed( conf.setMaximumCapacity(C, 70); // sub queues for A - conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A) + conf.set(QueuePrefixes.getQueuePrefix(A) + CapacitySchedulerConfiguration.QUEUES, "a1, a2 "); conf.setCapacity(A1, 60); @@ -448,7 +446,7 @@ public void testQueueParsing() throws Exception { public void testRootQueueParsing() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); // non-100 percent value will throw IllegalArgumentException - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT, 90); + csConf.setCapacity(ROOT, 90); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 55c43666cdb5b..8447261e6189e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.MappingRulesDescription; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; @@ -123,6 +124,14 @@ private static String prepareFileName(String f) { new File("src/test/resources/conversion-rules.properties") .getAbsolutePath(); + private static final QueuePath ROOT = new QueuePath("root"); + private static final QueuePath DEFAULT = new QueuePath("root.default"); + private static final QueuePath USERS = new QueuePath("root.users"); + private static final QueuePath USERS_JOE = new QueuePath("root.users.joe"); + private static final QueuePath USERS_JOHN = new QueuePath("root.users.john"); + private static final QueuePath ADMINS_ALICE = new QueuePath("root.admins.alice"); + private static final QueuePath ADMINS_BOB = new QueuePath("root.admins.bob"); + private ConversionOptions createDefaultConversionOptions() { return new ConversionOptions(new DryRunResultHolder(), false); } @@ -177,13 +186,13 @@ public void testDefaultMaxAMShare() throws Exception { assertEquals("Default max AM share", 0.16f, maxAmShare, 0.0f); assertEquals("root.admins.alice max-am-resource-percent", 0.15f, - conf.getMaximumApplicationMasterResourcePerQueuePercent("root.admins.alice"), + conf.getMaximumApplicationMasterResourcePerQueuePercent(ADMINS_ALICE), 0.0f); //root.users.joe don’t have maximum-am-resource-percent set // so falling back to the global value assertEquals("root.users.joe maximum-am-resource-percent", 0.16f, - conf.getMaximumApplicationMasterResourcePerQueuePercent("root.users.joe"), + conf.getMaximumApplicationMasterResourcePerQueuePercent(USERS_JOE), 0.0f); } @@ -194,20 +203,20 @@ public void testDefaultUserLimitFactor() throws Exception { CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); assertEquals("root.users user-limit-factor", 1.0f, - conf.getUserLimitFactor("root.users"), 0.0f); + conf.getUserLimitFactor(USERS), 0.0f); assertEquals("root.users auto-queue-creation-v2.enabled", true, - conf.isAutoQueueCreationV2Enabled("root.users")); + conf.isAutoQueueCreationV2Enabled(USERS)); assertEquals("root.default user-limit-factor", -1.0f, - conf.getUserLimitFactor("root.default"), 0.0f); + conf.getUserLimitFactor(DEFAULT), 0.0f); assertEquals("root.users.joe user-limit-factor", -1.0f, - conf.getUserLimitFactor("root.users.joe"), 0.0f); + conf.getUserLimitFactor(USERS_JOE), 0.0f); assertEquals("root.admins.bob user-limit-factor", -1.0f, - conf.getUserLimitFactor("root.admins.bob"), 0.0f); + conf.getUserLimitFactor(ADMINS_BOB), 0.0f); assertEquals("root.admin.bob auto-queue-creation-v2.enabled", false, - conf.isAutoQueueCreationV2Enabled("root.admin.bob")); + conf.isAutoQueueCreationV2Enabled(ADMINS_BOB)); } @Test @@ -227,15 +236,15 @@ public void testDefaultMaxAMShareDisabled() throws Exception { // root.admins.bob is unset,so falling back to the global value assertEquals("root.admins.bob maximum-am-resource-percent", 1.0f, - conf.getMaximumApplicationMasterResourcePerQueuePercent("root.admins.bob"), 0.0f); + conf.getMaximumApplicationMasterResourcePerQueuePercent(ADMINS_BOB), 0.0f); // root.admins.alice 0.15 != -1.0 assertEquals("root.admins.alice max-am-resource-percent", 0.15f, - conf.getMaximumApplicationMasterResourcePerQueuePercent("root.admins.alice"), 0.0f); + conf.getMaximumApplicationMasterResourcePerQueuePercent(ADMINS_ALICE), 0.0f); // root.users.joe is unset,so falling back to the global value assertEquals("root.users.joe maximum-am-resource-percent", 1.0f, - conf.getMaximumApplicationMasterResourcePerQueuePercent("root.users.joe"), 0.0f); + conf.getMaximumApplicationMasterResourcePerQueuePercent(USERS_JOE), 0.0f); } @Test @@ -246,33 +255,33 @@ public void testConvertACLs() throws Exception { // root assertEquals("root submit ACL", "alice,bob,joe,john hadoop_users", - conf.getAcl("root", QueueACL.SUBMIT_APPLICATIONS).getAclString()); + conf.getAcl(ROOT, QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root admin ACL", "alice,bob,joe,john hadoop_users", - conf.getAcl("root", QueueACL.ADMINISTER_QUEUE).getAclString()); + conf.getAcl(ROOT, QueueACL.ADMINISTER_QUEUE).getAclString()); // root.admins.bob assertEquals("root.admins.bob submit ACL", "bob ", - conf.getAcl("root.admins.bob", QueueACL.SUBMIT_APPLICATIONS).getAclString()); + conf.getAcl(ADMINS_BOB, QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root.admins.bob admin ACL", "bob ", - conf.getAcl("root.admins.bob", QueueACL.ADMINISTER_QUEUE).getAclString()); + conf.getAcl(ADMINS_BOB, QueueACL.ADMINISTER_QUEUE).getAclString()); // root.admins.alice assertEquals("root.admins.alice submit ACL", "alice ", - conf.getAcl("root.admins.alice", QueueACL.SUBMIT_APPLICATIONS).getAclString()); + conf.getAcl(ADMINS_ALICE, QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root.admins.alice admin ACL", "alice ", - conf.getAcl("root.admins.alice", QueueACL.ADMINISTER_QUEUE).getAclString()); + conf.getAcl(ADMINS_ALICE, QueueACL.ADMINISTER_QUEUE).getAclString()); // root.users.john assertEquals("root.users.john submit ACL", "*", - conf.getAcl("root.users.john", QueueACL.SUBMIT_APPLICATIONS).getAclString()); + conf.getAcl(USERS_JOHN, QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root.users.john admin ACL", "*", - conf.getAcl("root.users.john", QueueACL.ADMINISTER_QUEUE).getAclString()); + conf.getAcl(USERS_JOHN, QueueACL.ADMINISTER_QUEUE).getAclString()); // root.users.joe assertEquals("root.users.joe submit ACL", "joe ", - conf.getAcl("root.users.joe", QueueACL.SUBMIT_APPLICATIONS).getAclString()); + conf.getAcl(USERS_JOE, QueueACL.SUBMIT_APPLICATIONS).getAclString()); assertEquals("root.users.joe admin ACL", "joe ", - conf.getAcl("root.users.joe", QueueACL.ADMINISTER_QUEUE).getAclString()); + conf.getAcl(USERS_JOE, QueueACL.ADMINISTER_QUEUE).getAclString()); } @Test @@ -292,7 +301,7 @@ public void testSpecificQueueMaxParallelApps() throws Exception { CapacitySchedulerConfiguration conf = converter.getCapacitySchedulerConfig(); assertEquals("root.admins.alice max parallel apps", 2, - conf.getMaxParallelAppsForQueue("root.admins.alice"), 0); + conf.getMaxParallelAppsForQueue(ADMINS_ALICE), 0); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java index edab194a91817..ac3ce47633042 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -28,6 +28,7 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -93,6 +94,18 @@ private static String prepareFileName(String f) { private FSQueueConverterBuilder builder; private String key; + private static final QueuePath ROOT = new QueuePath("root"); + private static final QueuePath DEFAULT = new QueuePath("root.default"); + private static final QueuePath USERS = new QueuePath("root.users"); + private static final QueuePath USERS_JOE = new QueuePath("root.users.joe"); + private static final QueuePath USERS_JOHN = new QueuePath("root.users.john"); + private static final QueuePath ADMINS = new QueuePath("root.admins"); + private static final QueuePath ADMINS_ALICE = new QueuePath("root.admins.alice"); + private static final QueuePath ADMINS_BOB = new QueuePath("root.admins.bob"); + private static final QueuePath MISC = new QueuePath("root.misc"); + private static final QueuePath MISC_A = new QueuePath("root.misc.a"); + private static final QueuePath MISC_B = new QueuePath("root.misc.b"); + @Mock private FSConfigToCSConfigRuleHandler ruleHandler; @@ -155,20 +168,20 @@ public void testConvertQueueHierarchy() { converter.convertQueueHierarchy(rootQueue); // root children - assertArrayEquals("root children", new String[]{"admins", "users", "misc", "default"}, - csConfig.getQueues("root")); + assertEquals("root children", Arrays.asList("admins", "users", "misc", "default"), + csConfig.getQueues(ROOT)); // root.admins children - assertArrayEquals("root.admins children", new String[]{"bob", "alice"}, - csConfig.getQueues("root.admins")); + assertEquals("root.admins children", Arrays.asList("bob", "alice"), + csConfig.getQueues(ADMINS)); // root.default children - none - assertNull("root.default children", - csConfig.getQueues("root.default")); + assertTrue("root.default children", + csConfig.getQueues(DEFAULT).isEmpty()); // root.users children - assertArrayEquals("root.users children", new String[]{"john", "joe"}, - csConfig.getQueues("root.users")); + assertEquals("root.users children", Arrays.asList("john", "joe"), + csConfig.getQueues(USERS)); Set leafs = Sets.difference(ALL_QUEUES, Sets.newHashSet("root", @@ -179,8 +192,8 @@ public void testConvertQueueHierarchy() { for (String queue : leafs) { key = PREFIX + queue + ".queues"; - assertNull("Key " + key + " has value, but it should be null", - csConfig.getQueues(queue)); + assertTrue("Key " + key + " has value, but it should be empty", + csConfig.getQueues(new QueuePath(queue)).isEmpty()); } } @@ -194,12 +207,12 @@ public void testQueueMaxAMShare() { // root.admins.bob assertEquals("root.admins.bob AM share", 1.0f, csConfig.getMaximumApplicationMasterResourcePerQueuePercent( - "root.admins.bob"), 0.0f); + ADMINS_BOB), 0.0f); // root.admins.alice assertEquals("root.admins.alice AM share", 0.15f, csConfig.getMaximumApplicationMasterResourcePerQueuePercent( - "root.admins.alice"), 0.0f); + ADMINS_ALICE), 0.0f); Set remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins.bob", "root.admins.alice")); @@ -208,7 +221,7 @@ public void testQueueMaxAMShare() { key = PREFIX + queue + ".maximum-am-resource-percent"; assertEquals("Key " + key + " has different value", 0.1f, csConfig - .getMaximumApplicationMasterResourcePerQueuePercent(queue), 0.0f); + .getMaximumApplicationMasterResourcePerQueuePercent(new QueuePath(queue)), 0.0f); } } @@ -219,7 +232,7 @@ public void testQueueMaxParallelApps() { converter.convertQueueHierarchy(rootQueue); assertEquals("root.admins.alice max apps", 2, - csConfig.getMaxParallelAppsForQueue("root.admins.alice"), 0); + csConfig.getMaxParallelAppsForQueue(ADMINS_ALICE), 0); Set remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins.alice")); @@ -228,7 +241,7 @@ public void testQueueMaxParallelApps() { key = PREFIX + queue + ".max-parallel-apps"; assertEquals("Key " + key + " has different value", DEFAULT_MAX_PARALLEL_APPLICATIONS, csConfig - .getMaxParallelAppsForQueue(queue), 0); + .getMaxParallelAppsForQueue(new QueuePath(queue)), 0); } } @@ -240,15 +253,15 @@ public void testQueueMaxAllocations() { // root.admins vcores + mb assertEquals("root.admins max vcores", 3, - csConfig.getQueueMaximumAllocationVcores("root.admins")); + csConfig.getQueueMaximumAllocationVcores(ADMINS)); assertEquals("root.admins max memory", 4096, - csConfig.getQueueMaximumAllocationMb("root.admins")); + csConfig.getQueueMaximumAllocationMb(ADMINS)); // root.users.john max vcores + mb assertEquals("root.users.john max vcores", 2, - csConfig.getQueueMaximumAllocationVcores("root.users.john")); + csConfig.getQueueMaximumAllocationVcores(USERS_JOHN)); assertEquals("root.users.john max memory", 8192, - csConfig.getQueueMaximumAllocationMb("root.users.john")); + csConfig.getQueueMaximumAllocationMb(USERS_JOHN)); Set remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins", "root.users.john")); @@ -257,12 +270,12 @@ public void testQueueMaxAllocations() { key = PREFIX + queue + ".maximum-allocation-vcores"; assertEquals("Key " + key + " has different value", -1.0, csConfig - .getQueueMaximumAllocationVcores(queue), 0.0f); + .getQueueMaximumAllocationVcores(new QueuePath(queue)), 0.0f); key = PREFIX + queue + ".maximum-allocation-mb"; assertEquals("Key " + key + " has different value", -1.0, csConfig - .getQueueMaximumAllocationMb(queue), 0.0f); + .getQueueMaximumAllocationMb(new QueuePath(queue)), 0.0f); } } @@ -274,10 +287,10 @@ public void testQueuePreemptionDisabled() { assertTrue("root.admins.alice preemption setting", csConfig.getPreemptionDisabled( - "root.admins.alice", false)); + ADMINS_ALICE, false)); assertTrue("root.users.joe preemption setting", csConfig.getPreemptionDisabled( - "root.users.joe", false)); + USERS_JOE, false)); Set remaining = Sets.difference(ALL_QUEUES, Sets.newHashSet("root.admins.alice", "root.users.joe")); @@ -285,7 +298,7 @@ public void testQueuePreemptionDisabled() { for (String queue : remaining) { key = PREFIX + queue + ".disable_preemption"; assertEquals("Key " + key + " has different value", - false, csConfig.getPreemptionDisabled(queue, false)); + false, csConfig.getPreemptionDisabled(new QueuePath(queue), false)); } } @@ -298,7 +311,7 @@ public void testQueuePreemptionDisabledWhenGlobalPreemptionDisabled() { for (String queue : ALL_QUEUES) { key = PREFIX + queue + ".disable_preemption"; assertEquals("Key " + key + " has different value", - false, csConfig.getPreemptionDisabled(queue, false)); + false, csConfig.getPreemptionDisabled(new QueuePath(queue), false)); } } @@ -310,41 +323,31 @@ public void testChildCapacityInCapacityMode() { // root assertEquals("root.default capacity", 33.333f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.default")), 0.0f); + csConfig.getNonLabeledQueueCapacity(DEFAULT), 0.0f); assertEquals("root.admins capacity", 33.333f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.admins")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ADMINS), 0.0f); assertEquals("root.users capacity", 33.334f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.users")), 0.0f); + csConfig.getNonLabeledQueueCapacity(USERS), 0.0f); // root.users assertEquals("root.users.john capacity", 25.000f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.users.john")), 0.0f); + csConfig.getNonLabeledQueueCapacity(USERS_JOHN), 0.0f); assertEquals("root.users.joe capacity", 75.000f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.users.joe")), 0.0f); + csConfig.getNonLabeledQueueCapacity(USERS_JOE), 0.0f); // root.admins assertEquals("root.admins.alice capacity", 75.000f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.admins.alice")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ADMINS_ALICE), 0.0f); assertEquals("root.admins.bob capacity", 25.000f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.admins.bob")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ADMINS_BOB), 0.0f); // root.misc assertEquals("root.misc capacity", 0.000f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.misc")), 0.000f); + csConfig.getNonLabeledQueueCapacity(MISC), 0.000f); assertEquals("root.misc.a capacity", 0.000f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.misc.a")), 0.000f); + csConfig.getNonLabeledQueueCapacity(MISC_A), 0.000f); assertEquals("root.misc.b capacity", 0.000f, - csConfig.getNonLabeledQueueCapacity( - new QueuePath("root.misc.b")), 0.000f); + csConfig.getNonLabeledQueueCapacity(MISC_B), 0.000f); } @Test @@ -355,31 +358,31 @@ public void testChildCapacityInWeightMode() { // root assertEquals("root.default weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root.default"), 0.01f); + csConfig.getNonLabeledQueueWeight(DEFAULT), 0.01f); assertEquals("root.admins weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root.admins"), 0.01f); + csConfig.getNonLabeledQueueWeight(ADMINS), 0.01f); assertEquals("root.users weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root.users"), 0.01f); + csConfig.getNonLabeledQueueWeight(USERS), 0.01f); // root.users assertEquals("root.users.john weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root.users.john"), 0.01f); + csConfig.getNonLabeledQueueWeight(USERS_JOHN), 0.01f); assertEquals("root.users.joe weight", 3.0f, - csConfig.getNonLabeledQueueWeight("root.users.joe"), 0.01f); + csConfig.getNonLabeledQueueWeight(USERS_JOE), 0.01f); // root.admins assertEquals("root.admins.alice weight", 3.0f, - csConfig.getNonLabeledQueueWeight("root.admins.alice"), 0.01f); + csConfig.getNonLabeledQueueWeight(ADMINS_ALICE), 0.01f); assertEquals("root.admins.bob weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root.admins.bob"), 0.01f); + csConfig.getNonLabeledQueueWeight(ADMINS_BOB), 0.01f); // root.misc assertEquals("root.misc weight", 0.0f, - csConfig.getNonLabeledQueueWeight("root.misc"), 0.00f); + csConfig.getNonLabeledQueueWeight(MISC), 0.00f); assertEquals("root.misc.a weight", 0.0f, - csConfig.getNonLabeledQueueWeight("root.misc.a"), 0.00f); + csConfig.getNonLabeledQueueWeight(MISC_A), 0.00f); assertEquals("root.misc.b weight", 0.0f, - csConfig.getNonLabeledQueueWeight("root.misc.b"), 0.00f); + csConfig.getNonLabeledQueueWeight(MISC_B), 0.00f); } @Test @@ -389,15 +392,15 @@ public void testAutoCreateV2FlagsInWeightMode() { converter.convertQueueHierarchy(rootQueue); assertTrue("root autocreate v2 flag", - csConfig.isAutoQueueCreationV2Enabled("root")); + csConfig.isAutoQueueCreationV2Enabled(ROOT)); assertTrue("root.admins autocreate v2 flag", - csConfig.isAutoQueueCreationV2Enabled("root.admins")); + csConfig.isAutoQueueCreationV2Enabled(ADMINS)); assertTrue("root.admins.alice autocreate v2 flag", - csConfig.isAutoQueueCreationV2Enabled("root.admins.alice")); + csConfig.isAutoQueueCreationV2Enabled(ADMINS_ALICE)); assertTrue("root.users autocreate v2 flag", - csConfig.isAutoQueueCreationV2Enabled("root.users")); + csConfig.isAutoQueueCreationV2Enabled(USERS)); assertTrue("root.misc autocreate v2 flag", - csConfig.isAutoQueueCreationV2Enabled("root.misc")); + csConfig.isAutoQueueCreationV2Enabled(MISC)); //leaf queue root.admins.alice is removed from the below list //adding reservation to a leaf, it's queueType changes to FSParentQueue @@ -412,7 +415,7 @@ public void testAutoCreateV2FlagsInWeightMode() { key = PREFIX + queue + ".auto-queue-creation-v2.enabled"; assertEquals("Key " + key + " has different value", false, csConfig - .isAutoQueueCreationV2Enabled(queue)); + .isAutoQueueCreationV2Enabled(new QueuePath(queue))); } } @@ -430,11 +433,11 @@ public void testZeroSumCapacityValidation() { key = PREFIX + queue + ".allow-zero-capacity-sum"; assertEquals("Key " + key + " has different value", false, csConfig - .getAllowZeroCapacitySum(queue)); + .getAllowZeroCapacitySum(new QueuePath(queue))); } assertTrue("root.misc allow zero capacities", - csConfig.getAllowZeroCapacitySum("root.misc")); + csConfig.getAllowZeroCapacitySum(MISC)); } @Test @@ -472,7 +475,7 @@ public void testQueueWithNoAutoCreateChildQueue() { for (String queue : ALL_QUEUES) { key = PREFIX + queue + ".auto-create-child-queue.enabled"; assertEquals("Key " + key + " has different value", - false, csConfig.isAutoCreateChildQueueEnabled(queue)); + false, csConfig.isAutoCreateChildQueueEnabled(new QueuePath(queue))); } } @@ -517,25 +520,25 @@ public void testQueueOrderingPolicy() throws Exception { converter.convertQueueHierarchy(rootQueue); // root assertEquals("root ordering policy", "fifo", - csConfig.getAppOrderingPolicy("root").getConfigName()); + csConfig.getAppOrderingPolicy(ROOT).getConfigName()); assertEquals("root.default ordering policy", "fair", - csConfig.getAppOrderingPolicy("root.default").getConfigName()); + csConfig.getAppOrderingPolicy(DEFAULT).getConfigName()); assertEquals("root.admins ordering policy", "fifo", - csConfig.getAppOrderingPolicy("root.admins").getConfigName()); + csConfig.getAppOrderingPolicy(ADMINS).getConfigName()); assertEquals("root.users ordering policy", "fifo", - csConfig.getAppOrderingPolicy("root.users").getConfigName()); + csConfig.getAppOrderingPolicy(USERS).getConfigName()); // root.users assertEquals("root.users.joe ordering policy", "fair", - csConfig.getAppOrderingPolicy("root.users.joe").getConfigName()); + csConfig.getAppOrderingPolicy(USERS_JOE).getConfigName()); assertEquals("root.users.john ordering policy", "fifo", - csConfig.getAppOrderingPolicy("root.users.john").getConfigName()); + csConfig.getAppOrderingPolicy(USERS_JOHN).getConfigName()); // root.admins assertEquals("root.admins.alice ordering policy", "fifo", - csConfig.getAppOrderingPolicy("root.admins.alice.").getConfigName()); + csConfig.getAppOrderingPolicy(ADMINS_ALICE).getConfigName()); assertEquals("root.admins.bob ordering policy", "fair", - csConfig.getAppOrderingPolicy("root.admins.bob").getConfigName()); + csConfig.getAppOrderingPolicy(ADMINS_BOB).getConfigName()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToPercentageConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToPercentageConverter.java index 54fadf4571f43..33b18734a0a20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToPercentageConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToPercentageConverter.java @@ -38,6 +38,10 @@ public class TestWeightToPercentageConverter extends WeightConverterTestBase { private WeightToPercentConverter converter; private CapacitySchedulerConfiguration csConfig; + public static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + public static final QueuePath ROOT_A = new QueuePath("root", "a"); + public static final QueuePath ROOT_B = new QueuePath("root", "b"); + public static final QueuePath ROOT_C = new QueuePath("root", "c"); @Before public void setup() { @@ -52,7 +56,7 @@ public void testSingleWeightConversion() { converter.convertWeightsForChildQueues(root, csConfig); assertFalse("Capacity zerosum allowed", - csConfig.getAllowZeroCapacitySum("root")); + csConfig.getAllowZeroCapacitySum(ROOT)); assertEquals("root.a capacity", 100.000f, csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); } @@ -76,11 +80,11 @@ public void testMultiWeightConversion() { csConfig.getPropsWithPrefix(PREFIX).size()); // this is no fixing - it's the result of BigDecimal rounding assertEquals("root.a capacity", 16.667f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_A), 0.0f); assertEquals("root.b capacity", 33.333f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_B), 0.0f); assertEquals("root.c capacity", 50.000f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.c")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_C), 0.0f); } @Test @@ -90,15 +94,15 @@ public void testMultiWeightConversionWhenOfThemIsZero() { converter.convertWeightsForChildQueues(root, csConfig); assertFalse("Capacity zerosum allowed", - csConfig.getAllowZeroCapacitySum("root")); + csConfig.getAllowZeroCapacitySum(ROOT)); assertEquals("Number of properties", 22, csConfig.getPropsWithPrefix(PREFIX).size()); assertEquals("root.a capacity", 0.000f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_A), 0.0f); assertEquals("root.b capacity", 50.000f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_B), 0.0f); assertEquals("root.c capacity", 50.000f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.c")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_C), 0.0f); } @Test @@ -110,13 +114,13 @@ public void testMultiWeightConversionWhenAllOfThemAreZero() { assertEquals("Number of properties", 23, csConfig.getPropsWithPrefix(PREFIX).size()); assertTrue("Capacity zerosum allowed", - csConfig.getAllowZeroCapacitySum("root")); + csConfig.getAllowZeroCapacitySum(ROOT)); assertEquals("root.a capacity", 0.000f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_A), 0.0f); assertEquals("root.b capacity", 0.000f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_B), 0.0f); assertEquals("root.c capacity", 0.000f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.c")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_C), 0.0f); } @Test @@ -128,11 +132,11 @@ public void testCapacityFixingWithThreeQueues() { assertEquals("Number of properties", 22, csConfig.getPropsWithPrefix(PREFIX).size()); assertEquals("root.a capacity", 33.334f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.a")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_A), 0.0f); assertEquals("root.b capacity", 33.333f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.b")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_B), 0.0f); assertEquals("root.c capacity", 33.333f, - csConfig.getNonLabeledQueueCapacity(new QueuePath("root.c")), 0.0f); + csConfig.getNonLabeledQueueCapacity(ROOT_C), 0.0f); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java index 904dfe8b1f3d8..38b69c66a2926 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.junit.Before; import org.junit.Test; @@ -34,6 +35,11 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase { private WeightToWeightConverter converter; private CapacitySchedulerConfiguration csConfig; + public static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT); + public static final QueuePath ROOT_A = new QueuePath("root", "a"); + public static final QueuePath ROOT_B = new QueuePath("root", "b"); + public static final QueuePath ROOT_C = new QueuePath("root", "c"); + @Before public void setup() { converter = new WeightToWeightConverter(); @@ -47,7 +53,7 @@ public void testNoChildQueueConversion() { converter.convertWeightsForChildQueues(root, csConfig); assertEquals("root weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root"), 0.0f); + csConfig.getNonLabeledQueueWeight(ROOT), 0.0f); assertEquals("Converted items", 21, csConfig.getPropsWithPrefix(PREFIX).size()); } @@ -58,9 +64,9 @@ public void testSingleWeightConversion() { converter.convertWeightsForChildQueues(root, csConfig); assertEquals("root weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root"), 0.0f); + csConfig.getNonLabeledQueueWeight(ROOT), 0.0f); assertEquals("root.a weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root.a"), 0.0f); + csConfig.getNonLabeledQueueWeight(ROOT_A), 0.0f); assertEquals("Number of properties", 22, csConfig.getPropsWithPrefix(PREFIX).size()); } @@ -74,13 +80,13 @@ public void testMultiWeightConversion() { assertEquals("Number of properties", 24, csConfig.getPropsWithPrefix(PREFIX).size()); assertEquals("root weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root"), 0.0f); + csConfig.getNonLabeledQueueWeight(ROOT), 0.0f); assertEquals("root.a weight", 1.0f, - csConfig.getNonLabeledQueueWeight("root.a"), 0.0f); + csConfig.getNonLabeledQueueWeight(ROOT_A), 0.0f); assertEquals("root.b weight", 2.0f, - csConfig.getNonLabeledQueueWeight("root.b"), 0.0f); + csConfig.getNonLabeledQueueWeight(ROOT_B), 0.0f); assertEquals("root.c weight", 3.0f, - csConfig.getNonLabeledQueueWeight("root.c"), 0.0f); + csConfig.getNonLabeledQueueWeight(ROOT_C), 0.0f); } @Test @@ -89,7 +95,7 @@ public void testAutoCreateV2FlagOnParent() { converter.convertWeightsForChildQueues(root, csConfig); assertTrue("root autocreate v2 enabled", - csConfig.isAutoQueueCreationV2Enabled("root")); + csConfig.isAutoQueueCreationV2Enabled(ROOT)); } @Test @@ -100,6 +106,6 @@ public void testAutoCreateV2FlagOnParentWithoutChildren() { assertEquals("Number of properties", 21, csConfig.getPropsWithPrefix(PREFIX).size()); assertTrue("root autocreate v2 enabled", - csConfig.isAutoQueueCreationV2Enabled("root")); + csConfig.isAutoQueueCreationV2Enabled(ROOT)); } } From 219cf4b26d1e30e1db597deaefc7f6efec0fac80 Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Thu, 17 Aug 2023 11:38:03 +0200 Subject: [PATCH 20/21] YARN-11041. Fixed blanks and checkstyle Change-Id: Idf99d51c16345b74b851e9f7883bce6da33f8fa7 --- .../scheduler/capacity/ManagedParentQueue.java | 4 ++-- .../scheduler/fair/converter/FSQueueConverter.java | 3 ++- .../resourcemanager/TestWorkPreservingRMRestart.java | 2 +- .../scheduler/capacity/TestQueueParsing.java | 8 ++++---- .../scheduler/fair/converter/TestFSQueueConverter.java | 1 - .../webapp/TestRMWebServicesCapacitySched.java | 1 - .../TestRMWebServicesCapacitySchedDynamicConfig.java | 1 - 7 files changed, 9 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java index 0b9079990f84c..fa749f077964e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java @@ -39,7 +39,6 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_CAPACITY; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueueCapacityConfigParser; @@ -355,7 +354,8 @@ private void setLeafQueuesCapacityVector(AutoCreatedLeafQueue leafQueue) { for (String label : templateConfiguredNodeLabels) { final String leafConfigPath = QueuePrefixes.getNodeLabelPrefix( - QueuePrefixes.getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()), label); + QueuePrefixes.getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()), + label); String capacityString = leafConfig.get(leafConfigPath + CAPACITY, "0"); leafQueue.setConfiguredMinCapacityVector(label, getQueueCapacityConfigParser().parse(capacityString, leafQueue.getQueuePathObject())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index 5eacb34fa450b..510705ce14cd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -105,7 +105,8 @@ private void emitChildQueues(String queueName, List children) { List childQueues = children.stream() .map(child -> getQueueShortName(child.getName())) .collect(Collectors.toList()); - capacitySchedulerConfig.setQueues(new QueuePath(queueName), childQueues.toArray(new String[0])); + capacitySchedulerConfig.setQueues(new QueuePath(queueName), + childQueues.toArray(new String[0])); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index e91205d60bd13..76cf0db9a02b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -1680,7 +1680,7 @@ public void testQueueRecoveryOnRMWorkPreservingRestart() throws Exception { csConf.setCapacity(joe, 25); csConf.setCapacity(john, 25); csConf.setCapacity(defaultPath, 50); - + csConf.setQueues(joe, new String[] {"test"}); csConf.setQueues(john, new String[] {"test"}); csConf.setCapacity(new QueuePath(joe.getFullPath(), "test"), 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index f5099aa5bd692..65bce5c9318a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -841,8 +841,8 @@ public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard public void testQueueParsingWithMoveQueue() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setQueues(ROOT, new String[] { "a" }); - csConf.setQueues(A, new String[] { "x", "y" }); + csConf.setQueues(ROOT, new String[] {"a"}); + csConf.setQueues(A, new String[] {"x", "y"}); csConf.setCapacity(A, 100); csConf.setCapacity(AX, 50); csConf.setCapacity(AY, 50); @@ -851,8 +851,8 @@ public void testQueueParsingWithMoveQueue() throws Exception { MockRM rm = createMockRMWithoutLabels(conf); CapacityScheduler capacityScheduler = (CapacityScheduler) rm.getResourceScheduler(); - csConf.setQueues(ROOT, new String[] { "a", "x" }); - csConf.setQueues(A, new String[] { "y" }); + csConf.setQueues(ROOT, new String[] {"a", "x"}); + csConf.setQueues(A, new String[] {"y"}); csConf.setCapacity(X, 50); csConf.setCapacity(A, 50); csConf.setCapacity(AY, 100); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java index ac3ce47633042..47f8e1edfec3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -18,7 +18,6 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DEFAULT_MAX_PARALLEL_APPLICATIONS; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 3fe72f9f103b8..f60d29cb258e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.junit.runner.RunWith; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index e36a73a4e68d1..b1a6cdece8613 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -32,7 +32,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; From e8b8b9401b295d45c556392c417e67daaf9804ad Mon Sep 17 00:00:00 2001 From: Peter Szucs Date: Fri, 27 Oct 2023 11:39:49 +0200 Subject: [PATCH 21/21] YARN-11041. Fixed compilation error after merged trunk Change-Id: Id93e6ff990e9dad22362f4dc330783f7128a35f8 --- .../scheduler/capacity/TestCapacitySchedulerMultiNodes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java index f20cd1b275822..484308442ff31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java @@ -491,7 +491,7 @@ public void testSkipAllocationOnNodeReservedByAnotherApp() throws Exception { YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER); newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME + ".resource-based.sorting-interval.ms", 0); - newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default", 1.0f); + newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT, 1.0f); newConf.set(CapacitySchedulerConfiguration.SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS, "true"); MockRM rm1 = new MockRM(newConf);