diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index e3e157e451b..3a48bcb2f4a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; @@ -799,7 +798,7 @@ public void killAllAppsInQueue(String queueName) writeLock.unlock(); } } - + /** * Process resource update on a node. */ @@ -902,12 +901,12 @@ public void setClusterMaxPriority(Configuration conf) LOG.info("Updated the cluste max priority to maxClusterLevelAppPriority = " + maxClusterLevelAppPriority); } - + /** * Sanity check increase/decrease request, and return * SchedulerContainerResourceChangeRequest according to given * UpdateContainerRequest. - * + * *
* - Returns non-null value means validation succeeded
* - Throw exception when any other error happens
@@ -1344,9 +1343,7 @@ public void asyncContainerRelease(RMContainer container) {
}
/*
- * Get a Resource object with for the minimum allocation possible. If resource
- * profiles are enabled then the 'minimum' resource profile will be used. If
- * they are not enabled, use the minimums specified in the config files.
+ * Get a Resource object with for the minimum allocation possible.
*
* @return a Resource object with the minimum allocation for the scheduler
*/
@@ -1357,9 +1354,7 @@ public Resource getMinimumAllocation() {
}
/**
- * Get a Resource object with for the maximum allocation possible. If resource
- * profiles are enabled then the 'maximum' resource profile will be used. If
- * they are not enabled, use the maximums specified in the config files.
+ * Get a Resource object with for the maximum allocation possible.
*
* @return a Resource object with the maximum allocation for the scheduler
*/
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index 45d94919938..b1897547e01 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -47,14 +47,14 @@
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.eclipse.jetty.util.log.Log;
public class MockNM {
private int responseId;
private NodeId nodeId;
- private long memory;
- private int vCores;
+ private Resource capatibility;
private ResourceTrackerService resourceTracker;
private int httpPort = 2;
private MasterKey currentContainerTokenMasterKey;
@@ -75,13 +75,25 @@ public MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTrack
public MockNM(String nodeIdStr, int memory, int vcores,
ResourceTrackerService resourceTracker) {
- this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.getVersion());
+ this(nodeIdStr, memory, vcores, resourceTracker,
+ YarnVersionInfo.getVersion());
}
public MockNM(String nodeIdStr, int memory, int vcores,
ResourceTrackerService resourceTracker, String version) {
- this.memory = memory;
- this.vCores = vcores;
+ this(nodeIdStr, Resource.newInstance(memory, vcores), resourceTracker,
+ version);
+ }
+
+ public MockNM(String nodeIdStr, Resource capatibility,
+ ResourceTrackerService resourceTracker) {
+ this(nodeIdStr, capatibility, resourceTracker,
+ YarnVersionInfo.getVersion());
+ }
+
+ public MockNM(String nodeIdStr, Resource capatibility,
+ ResourceTrackerService resourceTracker, String version) {
+ this.capatibility = capatibility;
this.resourceTracker = resourceTracker;
this.version = version;
String[] splits = nodeIdStr.split(":");
@@ -146,8 +158,7 @@ public RegisterNodeManagerResponse registerNode(
RegisterNodeManagerRequest.class);
req.setNodeId(nodeId);
req.setHttpPort(httpPort);
- Resource resource = BuilderUtils.newResource(memory, vCores);
- req.setResource(resource);
+ req.setResource(capatibility);
req.setContainerStatuses(containerReports);
req.setNMVersion(version);
req.setRunningApplications(runningApplications);
@@ -158,8 +169,7 @@ public RegisterNodeManagerResponse registerNode(
this.currentNMTokenMasterKey = registrationResponse.getNMTokenMasterKey();
Resource newResource = registrationResponse.getResource();
if (newResource != null) {
- memory = (int) newResource.getMemorySize();
- vCores = newResource.getVirtualCores();
+ capatibility = Resources.clone(newResource);
}
containerStats.clear();
if (containerReports != null) {
@@ -185,7 +195,7 @@ public NodeHeartbeatResponse nodeHeartbeat(ApplicationAttemptId attemptId,
long containerId, ContainerState containerState) throws Exception {
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
BuilderUtils.newContainerId(attemptId, containerId), containerState,
- "Success", 0, BuilderUtils.newResource(memory, vCores));
+ "Success", 0, capatibility);
ArrayList containerStatusList =
new ArrayList(1);
containerStatusList.add(containerStatus);
@@ -266,19 +276,22 @@ public NodeHeartbeatResponse nodeHeartbeat(List updatedStats,
Resource newResource = heartbeatResponse.getResource();
if (newResource != null) {
- memory = newResource.getMemorySize();
- vCores = newResource.getVirtualCores();
+ capatibility = Resources.clone(newResource);
}
return heartbeatResponse;
}
public long getMemory() {
- return memory;
+ return capatibility.getMemorySize();
}
public int getvCores() {
- return vCores;
+ return capatibility.getVirtualCores();
+ }
+
+ public Resource getCapatibility() {
+ return capatibility;
}
public String getVersion() {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 1da93d15352..40ab17875e7 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -849,6 +849,15 @@ public MockNM registerNode(String nodeIdStr, int memory, int vCores,
return nm;
}
+ public MockNM registerNode(String nodeIdStr, Resource nodeCapatibility)
+ throws Exception {
+ MockNM nm = new MockNM(nodeIdStr, nodeCapatibility,
+ getResourceTrackerService());
+ nm.registerNode();
+ drainEventsImplicitly();
+ return nm;
+ }
+
public void sendNodeStarted(MockNM nm) throws Exception {
RMNodeImpl node = (RMNodeImpl) getRMContext().getRMNodes().get(
nm.getNodeId());
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 4bc5127e9da..8036a409df8 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -18,16 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-
+import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -38,6 +29,7 @@
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.Event;
@@ -53,21 +45,28 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-import com.google.common.collect.Sets;
-import org.apache.hadoop.yarn.event.Event;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
public class TestUtils {
private static final Log LOG = LogFactory.getLog(TestUtils.class);
@@ -457,4 +456,21 @@ public FiCaSchedulerApp getApplicationAttempt(
cs.submitResourceCommitRequest(clusterResource,
csAssignment);
}
+
+ /**
+ * An easy way to create resources other than memory and vcores for tests.
+ * @param memory memory
+ * @param vcores vcores
+ * @param nameToValues resource types other than memory and vcores.
+ * @return created resource
+ */
+ public static Resource createResource(long memory, int vcores,
+ Map nameToValues) {
+ Resource res = Resource.newInstance(memory, vcores);
+ for (Map.Entry entry : nameToValues.entrySet()) {
+ res.setResourceInformation(entry.getKey(), ResourceInformation
+ .newInstance(entry.getKey(), "", entry.getValue()));
+ }
+ return res;
+ }
}