diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java index 7ea7e811f35..cc110b000af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java @@ -25,6 +25,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.io.StringReader; import java.util.ArrayList; import java.util.EnumSet; @@ -36,13 +39,15 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import org.apache.hadoop.conf.Configuration; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceUtilization; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; @@ -56,16 +61,24 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.util.RackResolver; import org.apache.hadoop.yarn.util.YarnVersionInfo; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.w3c.dom.Document; @@ -86,6 +99,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase { private static MockRM rm; + private static File dest; private static class WebServletModule extends ServletModule { @Override @@ -93,7 +107,26 @@ protected void configureServlets() { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); - rm = new MockRM(new Configuration()); + + YarnConfiguration conf = new YarnConfiguration(); + + conf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class, ResourceCalculator.class); + + try { + ResourceUtils.resetResourceTypes(); + String resourceTypesFile = "resource-types-4.xml"; + InputStream source = + conf.getClassLoader().getResourceAsStream(resourceTypesFile); + dest = new File(conf.getClassLoader().getResource(".").getPath(), + "resource-types.xml"); + FileUtils.copyInputStreamToFile(source, dest); + ResourceUtils.getResourceTypes(); + } catch (IOException e) { + e.printStackTrace(); + } + + rm = new MockRM(conf); rm.getRMContext().getContainerTokenSecretManager().rollMasterKey(); rm.getRMContext().getNMTokenSecretManager().rollMasterKey(); rm.disableDrainEventsImplicitly(); @@ -115,6 +148,13 @@ public void setUp() throws Exception { Guice.createInjector(new WebServletModule())); } + @After + public void tearDown() throws IOException { + if (dest.exists()) { + dest.delete(); + } + } + public TestRMWebServicesNodes() { super(new WebAppDescriptor.Builder( "org.apache.hadoop.yarn.server.resourcemanager.webapp") @@ -912,4 +952,82 @@ private void verifyNodeAllocationTag(JSONObject json, } } + @Test + public void testNodesResourceTypes() throws JSONException, Exception { + + NodeId nm1 = NodeId.newInstance("host1", 1234); + NodeId nm2 = NodeId.newInstance("host2", 2345); + + rm.start(); + + Resource nm1Resource = Resource.newInstance(1024, 1); + nm1Resource.setResourceInformation("resource1", + ResourceInformation.newInstance("resource1", "G", 5)); + nm1Resource.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 6144)); + rm.registerNode(nm1.toString(), nm1Resource); + + Resource nm2Resource = Resource.newInstance(2024, 2); + nm2Resource.setResourceInformation("resource1", + ResourceInformation.newInstance("resource1", "G", 6)); + nm2Resource.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 8192)); + rm.registerNode(nm2.toString(), nm2Resource); + + Map expectedResourceTypes = new TreeMap<>(); + expectedResourceTypes.put(nm1.toString(), nm1Resource); + expectedResourceTypes.put(nm2.toString(), nm2Resource); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("nodes").accept("application/json").get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + JSONObject nodesInfoJson = response.getEntity(JSONObject.class); + verifyNodeResourceTypes(nodesInfoJson, expectedResourceTypes); + + rm.stop(); + } + + private void verifyNodeResourceTypes(JSONObject json, + Map expectedResourceTypes) + throws JSONException, JsonParseException, JsonMappingException, + IOException { + + ObjectMapper objectMapper = new ObjectMapper(); + String[] resources = { "memory-mb", "vcores", "resource1", "resource2" }; + + JSONArray nodes = json.getJSONObject("nodes").getJSONArray("node"); + assertEquals(expectedResourceTypes.size(), nodes.length()); + for (int i=0; i