diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 88b57f1..5d506a8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.records; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; @@ -54,9 +55,16 @@ @Public @Stable public static Resource newInstance(int memory, int vCores) { + return newInstance(memory, vCores, 0); + } + + @Private + @Evolving + public static Resource newInstance(int memory, int vCores, int vDisks) { Resource resource = Records.newRecord(Resource.class); resource.setMemory(memory); resource.setVirtualCores(vCores); + resource.setVirtualDisks(vDisks); return resource; } @@ -105,12 +113,31 @@ public static Resource newInstance(int memory, int vCores) { @Evolving public abstract void setVirtualCores(int vCores); + /** + * Get number of virtual disk I/O of the resource. + * + * @return number of virtual disk I/O of the resource + */ + @Private + @Evolving + public abstract int getVirtualDisks(); + + /** + * Set number of virtual disk I/O of the resource. + * + * @param vDisks number of virtual disk I/O of the resource + */ + @Private + @Evolving + public abstract void setVirtualDisks(int vDisks); + @Override public int hashCode() { final int prime = 263167; int result = 3571; result = 939769357 + getMemory(); // prime * result = 939769357 initially result = prime * result + getVirtualCores(); + result = prime * result + getVirtualDisks(); return result; } @@ -124,7 +151,8 @@ public boolean equals(Object obj) { return false; Resource other = (Resource) obj; if (getMemory() != other.getMemory() || - getVirtualCores() != other.getVirtualCores()) { + getVirtualCores() != other.getVirtualCores() || + getVirtualDisks() != other.getVirtualDisks()) { return false; } return true; @@ -132,6 +160,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return ""; + return ""; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 55073c5..0314e8c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -160,7 +160,14 @@ private static void addDeprecatedKeys() { public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = YARN_PREFIX + "scheduler.minimum-allocation-vcores"; - public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; + public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; + /** Now we limit the vdisks for each container as 1. This will be updated + * once we support the various vdisks requests. */ + @Private + public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS = + YARN_PREFIX + "scheduler.minimum-allocation-disk-vdisks"; + @Private + public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS = 1; /** Maximum request grant-able by the RM scheduler. */ public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = @@ -169,6 +176,11 @@ private static void addDeprecatedKeys() { public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = YARN_PREFIX + "scheduler.maximum-allocation-vcores"; public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; + @Private + public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_DISK_VDISKS = + YARN_PREFIX + "scheduler.maximum-allocation-disk-vdisks"; + @Private + public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_DISK_VDISKS = 1; /** Number of threads to handle scheduler interface.*/ public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = @@ -793,7 +805,13 @@ private static void addDeprecatedKeys() { NM_PREFIX + "resource.percentage-physical-cpu-limit"; public static final int DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT = 100; - + + /** Number of virtual disk I/O resources which can be allocated for containers. */ + @Private + public static final String NM_DISK_VDISKS = NM_PREFIX + "resource.disk-vdisks"; + @Private + public static final int DEFAULT_NM_DISK_VDISKS = 20; + /** NM Webapp address.**/ public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; public static final int DEFAULT_NM_WEBAPP_PORT = 8042; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c4e756d..65a36b8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -56,6 +56,7 @@ message ContainerIdProto { message ResourceProto { optional int32 memory = 1; optional int32 virtual_cores = 2; + optional int32 virtual_disks = 3; } message ResourceOptionProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java index fa2779e..a793c8f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java @@ -202,6 +202,11 @@ private void printNodeStatus(String nodeIdStr) throws YarnException, : (nodeReport.getUsed().getVirtualCores() + " vcores")); nodeReportStr.print("\tCPU-Capacity : "); nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores"); + nodeReportStr.print("\tDisk-I/O-Used : "); + nodeReportStr.println((nodeReport.getUsed() == null) ? + "0 vdisks" : (nodeReport.getUsed().getVirtualDisks() + " vdisks")); + nodeReportStr.print("\tDisk-I/O-Capacity : "); + nodeReportStr.println(nodeReport.getCapability().getVirtualDisks() + " vdisks"); nodeReportStr.print("\tNode-Labels : "); // Create a List for node labels since we need it get sorted diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java index 8885769..6a7f8da 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestResourceTrackerOnHA.java @@ -55,7 +55,7 @@ public void shutDown() { @Test(timeout = 15000) public void testResourceTrackerOnHA() throws Exception { NodeId nodeId = NodeId.newInstance("localhost", 0); - Resource resource = Resource.newInstance(2048, 4); + Resource resource = Resource.newInstance(2048, 4, 4); // make sure registerNodeManager works when failover happens RegisterNodeManagerRequest request = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 194d7d1..08d3ae5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -1152,6 +1152,8 @@ public void testNodeStatus() throws Exception { pw.println("\tMemory-Capacity : 0MB"); pw.println("\tCPU-Used : 0 vcores"); pw.println("\tCPU-Capacity : 0 vcores"); + pw.println("\tDisk-I/O-Used : 0 vdisks"); + pw.println("\tDisk-I/O-Capacity : 0 vdisks"); pw.println("\tNode-Labels : a,b,c,x,y,z"); pw.close(); String nodeStatusStr = baos.toString("UTF-8"); @@ -1186,6 +1188,8 @@ public void testNodeStatusWithEmptyNodeLabels() throws Exception { pw.println("\tMemory-Capacity : 0MB"); pw.println("\tCPU-Used : 0 vcores"); pw.println("\tCPU-Capacity : 0 vcores"); + pw.println("\tDisk-I/O-Used : 0 vdisks"); + pw.println("\tDisk-I/O-Capacity : 0 vdisks"); pw.println("\tNode-Labels : "); pw.close(); String nodeStatusStr = baos.toString("UTF-8"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index a28c6ed..6f4cd86 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -80,10 +80,25 @@ public void setVirtualCores(int vCores) { } @Override + public int getVirtualDisks() { + ResourceProtoOrBuilder p = viaProto ? proto : builder; + return (p.getVirtualDisks()); + } + + @Override + public void setVirtualDisks(int vDisks) { + maybeInitBuilder(); + builder.setVirtualDisks((vDisks)); + } + + @Override public int compareTo(Resource other) { int diff = this.getMemory() - other.getMemory(); if (diff == 0) { diff = this.getVirtualCores() - other.getVirtualCores(); + if (diff == 0) { + diff = this.getVirtualDisks() - other.getVirtualDisks(); + } } return diff; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 6f5b40e..fbcc6b8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -21,6 +21,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import java.util.Arrays; + /** * A {@link ResourceCalculator} which uses the concept of * dominant resource to compare multi-dimensional resources. @@ -52,62 +54,61 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs) { if (lhs.equals(rhs)) { return 0; } - - float l = getResourceAsValue(clusterResource, lhs, true); - float r = getResourceAsValue(clusterResource, rhs, true); - - if (l < r) { - return -1; - } else if (l > r) { - return 1; - } else { - l = getResourceAsValue(clusterResource, lhs, false); - r = getResourceAsValue(clusterResource, rhs, false); + + for (int i = 1; i <= 3; i ++) { + float l = getResourceAsValue(clusterResource, lhs, i); + float r = getResourceAsValue(clusterResource, rhs, i); + if (l < r) { return -1; } else if (l > r) { return 1; } } - return 0; } /** - * Use 'dominant' for now since we only have 2 resources - gives us a slight - * performance boost. - * - * Once we add more resources, we'll need a more complicated (and slightly - * less performant algorithm). + * Get the resource value according to the rank. Lower the rank, more + * dominant the resource. Rank = 1 means the most dominant resource value. */ protected float getResourceAsValue( - Resource clusterResource, Resource resource, boolean dominant) { - // Just use 'dominant' resource - return (dominant) ? - Math.max( - (float)resource.getMemory() / clusterResource.getMemory(), - (float)resource.getVirtualCores() / clusterResource.getVirtualCores() - ) - : - Math.min( - (float)resource.getMemory() / clusterResource.getMemory(), - (float)resource.getVirtualCores() / clusterResource.getVirtualCores() - ); + Resource clusterResource, Resource resource, int rank) { + if (rank < 1 || rank > 3) { + throw new IllegalArgumentException("The rank " + rank + + " is not in range [1,3]."); + } + float[] values = new float[] { + (float) resource.getMemory() / clusterResource.getMemory(), + (float) resource.getVirtualCores() / clusterResource.getVirtualCores(), + (float) resource.getVirtualDisks() / clusterResource.getVirtualDisks()}; + Arrays.sort(values); + return values[values.length - rank]; } - + @Override public int computeAvailableContainers(Resource available, Resource required) { - return Math.min( - available.getMemory() / required.getMemory(), - available.getVirtualCores() / required.getVirtualCores()); + int min = Integer.MAX_VALUE; + if (required.getMemory() != 0) { + min = Math.min(min, + available.getMemory() / required.getMemory()); + } + if (required.getVirtualCores() != 0) { + min = Math.min(min, + available.getVirtualCores() / required.getVirtualCores()); + } + if (required.getVirtualDisks() != 0) { + min = Math.min(min, + available.getVirtualDisks() / required.getVirtualDisks()); + } + return min; } @Override public float divide(Resource clusterResource, Resource numerator, Resource denominator) { - return - getResourceAsValue(clusterResource, numerator, true) / - getResourceAsValue(clusterResource, denominator, true); + return getResourceAsValue(clusterResource, numerator, 1) / + getResourceAsValue(clusterResource, denominator, 1); } @Override @@ -120,17 +121,28 @@ public boolean isInvalidDivisor(Resource r) { @Override public float ratio(Resource a, Resource b) { - return Math.max( - (float)a.getMemory()/b.getMemory(), - (float)a.getVirtualCores()/b.getVirtualCores() - ); + float max = 0.0f; + if (b.getMemory() != 0) { + max = Math.max(max, + (float) a.getMemory() / b.getMemory()); + } + if (b.getVirtualCores() != 0) { + max = Math.max(max, + (float) a.getVirtualCores() / b.getVirtualCores()); + } + if (b.getVirtualDisks() != 0) { + max = Math.max(max, + (float) a.getVirtualDisks() / b.getVirtualDisks()); + } + return max; } @Override public Resource divideAndCeil(Resource numerator, int denominator) { return Resources.createResource( divideAndCeil(numerator.getMemory(), denominator), - divideAndCeil(numerator.getVirtualCores(), denominator) + divideAndCeil(numerator.getVirtualCores(), denominator), + divideAndCeil(numerator.getVirtualDisks(), denominator) ); } @@ -147,15 +159,22 @@ public Resource normalize(Resource r, Resource minimumResource, Math.max(r.getVirtualCores(), minimumResource.getVirtualCores()), stepFactor.getVirtualCores()), maximumResource.getVirtualCores()); + int normalizedVdisks = Math.min( + roundUp( + Math.max(r.getVirtualDisks(), minimumResource.getVirtualDisks()), + stepFactor.getVirtualDisks()), + maximumResource.getVirtualDisks()); + return Resources.createResource(normalizedMemory, - normalizedCores); + normalizedCores, normalizedVdisks); } @Override public Resource roundUp(Resource r, Resource stepFactor) { return Resources.createResource( roundUp(r.getMemory(), stepFactor.getMemory()), - roundUp(r.getVirtualCores(), stepFactor.getVirtualCores()) + roundUp(r.getVirtualCores(), stepFactor.getVirtualCores()), + roundUp(r.getVirtualDisks(), stepFactor.getVirtualDisks()) ); } @@ -163,7 +182,8 @@ public Resource roundUp(Resource r, Resource stepFactor) { public Resource roundDown(Resource r, Resource stepFactor) { return Resources.createResource( roundDown(r.getMemory(), stepFactor.getMemory()), - roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()) + roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()), + roundDown(r.getVirtualDisks(), stepFactor.getVirtualDisks()) ); } @@ -175,7 +195,10 @@ public Resource multiplyAndNormalizeUp(Resource r, double by, (int)Math.ceil(r.getMemory() * by), stepFactor.getMemory()), roundUp( (int)Math.ceil(r.getVirtualCores() * by), - stepFactor.getVirtualCores()) + stepFactor.getVirtualCores()), + roundDown( + (int)Math.ceil(r.getVirtualDisks() * by), + stepFactor.getVirtualDisks()) ); } @@ -190,6 +213,10 @@ public Resource multiplyAndNormalizeDown(Resource r, double by, roundDown( (int)(r.getVirtualCores() * by), stepFactor.getVirtualCores() + ), + roundDown( + (int)(r.getVirtualDisks() * by), + stepFactor.getVirtualDisks() ) ); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index a205bd1..bc6fd0e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -51,10 +51,23 @@ public void setVirtualCores(int cores) { } @Override + public int getVirtualDisks() { + return 0; + } + + @Override + public void setVirtualDisks(int disks) { + throw new RuntimeException("NONE cannot be modified!"); + } + + @Override public int compareTo(Resource o) { int diff = 0 - o.getMemory(); if (diff == 0) { diff = 0 - o.getVirtualCores(); + if (diff == 0) { + diff = 0 - o.getVirtualDisks(); + } } return diff; } @@ -70,7 +83,7 @@ public int getMemory() { @Override public void setMemory(int memory) { - throw new RuntimeException("NONE cannot be modified!"); + throw new RuntimeException("UNBOUNDED cannot be modified!"); } @Override @@ -80,7 +93,17 @@ public int getVirtualCores() { @Override public void setVirtualCores(int cores) { - throw new RuntimeException("NONE cannot be modified!"); + throw new RuntimeException("UNBOUNDED cannot be modified!"); + } + + @Override + public int getVirtualDisks() { + return Integer.MAX_VALUE; + } + + @Override + public void setVirtualDisks(int disks) { + throw new RuntimeException("UNBOUNDED cannot be modified!"); } @Override @@ -88,6 +111,9 @@ public int compareTo(Resource o) { int diff = 0 - o.getMemory(); if (diff == 0) { diff = 0 - o.getVirtualCores(); + if (diff == 0) { + diff = 0 - o.getVirtualDisks(); + } } return diff; } @@ -99,9 +125,14 @@ public static Resource createResource(int memory) { } public static Resource createResource(int memory, int cores) { + return createResource(memory, cores, (memory > 0) ? 1 : 0); + } + + public static Resource createResource(int memory, int cores, int vdisks) { Resource resource = Records.newRecord(Resource.class); resource.setMemory(memory); resource.setVirtualCores(cores); + resource.setVirtualDisks(vdisks); return resource; } @@ -114,12 +145,14 @@ public static Resource unbounded() { } public static Resource clone(Resource res) { - return createResource(res.getMemory(), res.getVirtualCores()); + return createResource( + res.getMemory(), res.getVirtualCores(), res.getVirtualDisks()); } public static Resource addTo(Resource lhs, Resource rhs) { lhs.setMemory(lhs.getMemory() + rhs.getMemory()); lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores()); + lhs.setVirtualDisks(lhs.getVirtualDisks() + rhs.getVirtualDisks()); return lhs; } @@ -130,6 +163,7 @@ public static Resource add(Resource lhs, Resource rhs) { public static Resource subtractFrom(Resource lhs, Resource rhs) { lhs.setMemory(lhs.getMemory() - rhs.getMemory()); lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); + lhs.setVirtualDisks(lhs.getVirtualDisks() - rhs.getVirtualDisks()); return lhs; } @@ -144,6 +178,7 @@ public static Resource negate(Resource resource) { public static Resource multiplyTo(Resource lhs, double by) { lhs.setMemory((int)(lhs.getMemory() * by)); lhs.setVirtualCores((int)(lhs.getVirtualCores() * by)); + lhs.setVirtualDisks((int)(lhs.getVirtualDisks() * by)); return lhs; } @@ -165,6 +200,7 @@ public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); out.setMemory((int)(lhs.getMemory() * by)); out.setVirtualCores((int)(lhs.getVirtualCores() * by)); + out.setVirtualDisks((int)(lhs.getVirtualDisks() * by)); return out; } @@ -253,11 +289,13 @@ public static Resource max( public static boolean fitsIn(Resource smaller, Resource bigger) { return smaller.getMemory() <= bigger.getMemory() && - smaller.getVirtualCores() <= bigger.getVirtualCores(); + smaller.getVirtualCores() <= bigger.getVirtualCores() && + smaller.getVirtualDisks() <= bigger.getVirtualDisks(); } public static Resource componentwiseMin(Resource lhs, Resource rhs) { return createResource(Math.min(lhs.getMemory(), rhs.getMemory()), - Math.min(lhs.getVirtualCores(), rhs.getVirtualCores())); + Math.min(lhs.getVirtualCores(), rhs.getVirtualCores()), + Math.min(lhs.getVirtualDisks(), rhs.getVirtualDisks())); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index 20983b6..542951d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -133,6 +133,7 @@ public void testRegisterNodeManagerRequestPBImpl() { Resource resource = recordFactory.newRecordInstance(Resource.class); resource.setMemory(10000); resource.setVirtualCores(2); + resource.setVirtualDisks(3); original.setResource(resource); RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl( original.getProto()); @@ -141,7 +142,7 @@ public void testRegisterNodeManagerRequestPBImpl() { assertEquals(9090, copy.getNodeId().getPort()); assertEquals(10000, copy.getResource().getMemory()); assertEquals(2, copy.getResource().getVirtualCores()); - + assertEquals(3, copy.getResource().getVirtualDisks()); } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index f561dbb..4b2e6c0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -142,8 +142,11 @@ protected void serviceInit(Configuration conf) throws Exception { int virtualCores = conf.getInt( YarnConfiguration.NM_VCORES, YarnConfiguration.DEFAULT_NM_VCORES); + int virtualDisks = + conf.getInt( + YarnConfiguration.NM_DISK_VDISKS, YarnConfiguration.DEFAULT_NM_DISK_VDISKS); - this.totalResource = Resource.newInstance(memoryMb, virtualCores); + this.totalResource = Resource.newInstance(memoryMb, virtualCores, virtualDisks); metrics.addResource(totalResource); this.tokenKeepAliveEnabled = isTokenKeepAliveEnabled(conf); this.tokenRemovalDelayMs = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 4beb895..dc505b1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -99,6 +99,7 @@ private int minAllocMb; private int minAllocVcores; + private int minAllocVdisks; static { resync.setNodeAction(NodeAction.RESYNC); @@ -144,6 +145,9 @@ protected void serviceInit(Configuration conf) throws Exception { minAllocVcores = conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + minAllocVdisks = conf.getInt( + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS); minimumNodeManagerVersion = conf.get( YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, @@ -285,7 +289,8 @@ public RegisterNodeManagerResponse registerNodeManager( // Check if this node has minimum allocations if (capability.getMemory() < minAllocMb - || capability.getVirtualCores() < minAllocVcores) { + || capability.getVirtualCores() < minAllocVcores + || capability.getVirtualDisks() < minAllocVdisks) { String message = "NodeManager from " + host + " doesn't satisfy minimum allocations, Sending SHUTDOWN" diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceType.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceType.java index 9dd245b..a6ebcf2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceType.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceType.java @@ -24,5 +24,5 @@ @Private @Evolving public enum ResourceType { - MEMORY, CPU + MEMORY, CPU, DISKIO } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java index 230f9a9..3ce7c0f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceWeights.java @@ -29,8 +29,13 @@ private float[] weights = new float[ResourceType.values().length]; public ResourceWeights(float memoryWeight, float cpuWeight) { + this(memoryWeight, cpuWeight, 0); + } + + public ResourceWeights(float memoryWeight, float cpuWeight, float vdiskWeight) { weights[ResourceType.MEMORY.ordinal()] = memoryWeight; weights[ResourceType.CPU.ordinal()] = cpuWeight; + weights[ResourceType.DISKIO.ordinal()] = vdiskWeight; } public ResourceWeights(float weight) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 5d00009..de8382a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -190,7 +190,7 @@ public static void normalizeRequest( /** * Utility method to validate a resource request, by insuring that the - * requested memory/vcore is non-negative and not greater than max + * requested memory/vcore/vdisks is non-negative and not greater than max * * @throws InvalidResourceRequestException when there is invalid * request @@ -216,7 +216,17 @@ public static void validateResourceRequest(ResourceRequest resReq, + resReq.getCapability().getVirtualCores() + ", maxVirtualCores=" + maximumResource.getVirtualCores()); } - + if (resReq.getCapability().getVirtualDisks() < 0 || + resReq.getCapability().getVirtualDisks() > + maximumResource.getVirtualDisks()) { + throw new InvalidResourceRequestException("Invalid resource request" + + ", requested virtual disks < 0" + + ", or requested virtual disks > max configured" + + ", requestedVirtualDisks=" + + resReq.getCapability().getVirtualDisks() + + ", maxVirtualDisks=" + maximumResource.getVirtualDisks()); + } + // Get queue from scheduler QueueInfo queueInfo = null; try { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java index 3bea985..c981e1b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java @@ -257,6 +257,8 @@ private static int getResourceValue(Resource resource, ResourceType type) { return resource.getMemory(); case CPU: return resource.getVirtualCores(); + case DISKIO: + return resource.getVirtualDisks(); default: throw new IllegalArgumentException("Invalid resource"); } @@ -270,6 +272,9 @@ private static void setResourceValue(int val, Resource resource, ResourceType ty case CPU: resource.setVirtualCores(val); break; + case DISKIO: + resource.setVirtualDisks(val); + break; default: throw new IllegalArgumentException("Invalid resource"); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index 3f6cbd1..30826b0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -96,10 +96,15 @@ public Resource getHeadroom(Resource queueFairShare, Resource queueUsage, int queueAvailableCPU = Math.max(queueFairShare.getVirtualCores() - queueUsage .getVirtualCores(), 0); + int queueAvailableVDisks = + Math.max(queueFairShare.getVirtualDisks() - queueUsage + .getVirtualDisks(), 0); Resource headroom = Resources.createResource( Math.min(clusterAvailable.getMemory(), queueAvailableMemory), Math.min(clusterAvailable.getVirtualCores(), - queueAvailableCPU)); + queueAvailableCPU), + Math.min(clusterAvailable.getVirtualDisks(), + queueAvailableVDisks)); return headroom; } @@ -174,14 +179,37 @@ void calculateShares(Resource resource, Resource pool, (pool.getMemory() * weights.getWeight(MEMORY))); shares.setWeight(CPU, (float)resource.getVirtualCores() / (pool.getVirtualCores() * weights.getWeight(CPU))); + shares.setWeight(DISKIO, (float)resource.getVirtualDisks() / + (pool.getVirtualDisks() * weights.getWeight(DISKIO))); // sort order vector by resource share if (resourceOrder != null) { if (shares.getWeight(MEMORY) > shares.getWeight(CPU)) { - resourceOrder[0] = MEMORY; - resourceOrder[1] = CPU; + if (shares.getWeight(MEMORY) > shares.getWeight(DISKIO)) { + resourceOrder[0] = MEMORY; + if (shares.getWeight(CPU) > shares.getWeight(DISKIO)) { + resourceOrder[1] = CPU; + resourceOrder[2] = DISKIO; + } + } else { + resourceOrder[0] = DISKIO; + resourceOrder[1] = MEMORY; + resourceOrder[2] = CPU; + } } else { - resourceOrder[0] = CPU; - resourceOrder[1] = MEMORY; + if (shares.getWeight(CPU) > shares.getWeight(DISKIO)) { + resourceOrder[0] = CPU; + if (shares.getWeight(MEMORY) > shares.getWeight(DISKIO)) { + resourceOrder[1] = MEMORY; + resourceOrder[2] = DISKIO; + } else { + resourceOrder[1] = DISKIO; + resourceOrder[2] = MEMORY; + } + } else { + resourceOrder[0] = DISKIO; + resourceOrder[1] = CPU; + resourceOrder[2] = MEMORY; + } } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java index 97669cb..50bcdb4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java @@ -121,7 +121,7 @@ public Resource getHeadroom(Resource queueFairShare, queueFairShare.getMemory() - queueUsage.getMemory(), 0); Resource headroom = Resources.createResource( Math.min(clusterAvailable.getMemory(), queueAvailableMemory), - clusterAvailable.getVirtualCores()); + clusterAvailable.getVirtualCores(), clusterAvailable.getVirtualDisks()); return headroom; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index 5f53805..6bfc0d2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -50,6 +50,7 @@ private NodeId nodeId; private final int memory; private final int vCores; + private final int vDisks; private ResourceTrackerService resourceTracker; private final int httpPort = 2; private MasterKey currentContainerTokenMasterKey; @@ -65,14 +66,33 @@ public MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTrack } public MockNM(String nodeIdStr, int memory, int vcores, + ResourceTrackerService resourceTracker) { + // scale vdisks based on the requested memory + this(nodeIdStr, memory, vcores, + Math.max(1, (memory * YarnConfiguration.DEFAULT_NM_DISK_VDISKS) / + YarnConfiguration.DEFAULT_NM_PMEM_MB), + resourceTracker); + } + + public MockNM(String nodeIdStr, int memory, int vcores, int vdisks, ResourceTrackerService resourceTracker) { - this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.getVersion()); + this(nodeIdStr, memory, vcores, vdisks, resourceTracker, + YarnVersionInfo.getVersion()); } public MockNM(String nodeIdStr, int memory, int vcores, ResourceTrackerService resourceTracker, String version) { + this(nodeIdStr, memory, vcores, + Math.max(1, (memory * YarnConfiguration.DEFAULT_NM_DISK_VDISKS) / + YarnConfiguration.DEFAULT_NM_PMEM_MB), + resourceTracker, version); + } + + public MockNM(String nodeIdStr, int memory, int vcores, int vDisks, + ResourceTrackerService resourceTracker, String version) { this.memory = memory; this.vCores = vcores; + this.vDisks = vDisks; this.resourceTracker = resourceTracker; this.version = version; String[] splits = nodeIdStr.split(":"); @@ -115,7 +135,7 @@ public RegisterNodeManagerResponse registerNode( RegisterNodeManagerRequest.class); req.setNodeId(nodeId); req.setHttpPort(httpPort); - Resource resource = BuilderUtils.newResource(memory, vCores); + Resource resource = Resource.newInstance(memory, vCores, vDisks); req.setResource(resource); req.setContainerStatuses(containerReports); req.setNMVersion(version); @@ -198,4 +218,8 @@ public int getMemory() { public int getvCores() { return vCores; } + + public int getVDisks() { + return vDisks; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 6b3eea2..da7494c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -574,7 +574,7 @@ public void testEscapeApplicationSummary() { when(app.getState()).thenReturn(RMAppState.RUNNING); RMAppMetrics metrics = - new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64); + new RMAppMetrics(Resource.newInstance(1234, 56, 1), 10, 1, 16384, 64); when(app.getRMAppMetrics()).thenReturn(metrics); RMAppManager.ApplicationSummary.SummaryBuilder summary = @@ -592,7 +592,8 @@ public void testEscapeApplicationSummary() { Assert.assertTrue(msg.contains("vcoreSeconds=64")); Assert.assertTrue(msg.contains("preemptedAMContainers=1")); Assert.assertTrue(msg.contains("preemptedNonAMContainers=10")); - Assert.assertTrue(msg.contains("preemptedResources=")); + Assert.assertTrue( + msg.contains("preemptedResources=")); } private static ResourceScheduler mockResourceScheduler() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 7c12848..3551b8b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -298,7 +298,7 @@ public void testNodeRegistrationSuccess() throws Exception { RegisterNodeManagerRequest req = Records.newRecord( RegisterNodeManagerRequest.class); NodeId nodeId = NodeId.newInstance("host2", 1234); - Resource capability = BuilderUtils.newResource(1024, 1); + Resource capability = Resource.newInstance(1024, 1, 1); req.setResource(capability); req.setNodeId(nodeId); req.setHttpPort(1234); @@ -382,6 +382,7 @@ public void testNodeRegistrationWithMinimumAllocations() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048"); conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4"); + conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS, "1"); rm = new MockRM(conf); rm.start(); @@ -392,7 +393,7 @@ public void testNodeRegistrationWithMinimumAllocations() throws Exception { NodeId nodeId = BuilderUtils.newNodeId("host", 1234); req.setNodeId(nodeId); - Resource capability = BuilderUtils.newResource(1024, 1); + Resource capability = Resource.newInstance(1024, 1, 1); req.setResource(capability); RegisterNodeManagerResponse response1 = resourceTrackerService.registerNodeManager(req); @@ -400,6 +401,7 @@ public void testNodeRegistrationWithMinimumAllocations() throws Exception { capability.setMemory(2048); capability.setVirtualCores(1); + capability.setVirtualDisks(1); req.setResource(capability); RegisterNodeManagerResponse response2 = resourceTrackerService.registerNodeManager(req); @@ -407,17 +409,27 @@ public void testNodeRegistrationWithMinimumAllocations() throws Exception { capability.setMemory(1024); capability.setVirtualCores(4); + capability.setVirtualDisks(1); req.setResource(capability); RegisterNodeManagerResponse response3 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction()); - + capability.setMemory(2048); capability.setVirtualCores(4); + capability.setVirtualDisks(0); req.setResource(capability); RegisterNodeManagerResponse response4 = resourceTrackerService.registerNodeManager(req); - Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction()); + Assert.assertEquals(NodeAction.SHUTDOWN,response4.getNodeAction()); + + capability.setMemory(2048); + capability.setVirtualCores(4); + capability.setVirtualDisks(1); + req.setResource(capability); + RegisterNodeManagerResponse response5 = + resourceTrackerService.registerNodeManager(req); + Assert.assertEquals(NodeAction.NORMAL,response5.getNodeAction()); } @Test @@ -512,7 +524,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { NMContainerStatus.newInstance( ContainerId.newContainerId( ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1), - ContainerState.COMPLETE, Resource.newInstance(1024, 1), + ContainerState.COMPLETE, Resource.newInstance(1024, 1, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234); rm.getResourceTrackerService().handleNMContainerStatus(report, null); verify(handler, never()).handle((Event) any()); @@ -523,7 +535,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { currentAttempt.setMasterContainer(null); report = NMContainerStatus.newInstance( ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0), - ContainerState.COMPLETE, Resource.newInstance(1024, 1), + ContainerState.COMPLETE, Resource.newInstance(1024, 1, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234); rm.getResourceTrackerService().handleNMContainerStatus(report, null); verify(handler, never()).handle((Event)any()); @@ -535,7 +547,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { report = NMContainerStatus.newInstance( ContainerId.newContainerId( ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1), - ContainerState.COMPLETE, Resource.newInstance(1024, 1), + ContainerState.COMPLETE, Resource.newInstance(1024, 1, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234); try { rm.getResourceTrackerService().handleNMContainerStatus(report, null); @@ -550,7 +562,7 @@ public void testHandleContainerStatusInvalidCompletions() throws Exception { currentAttempt.setMasterContainer(null); report = NMContainerStatus.newInstance( ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0), - ContainerState.COMPLETE, Resource.newInstance(1024, 1), + ContainerState.COMPLETE, Resource.newInstance(1024, 1, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234); try { rm.getResourceTrackerService().handleNMContainerStatus(report, null); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 853e0a5..0746ef8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -227,7 +227,7 @@ public void testSchedulerRecovery() throws Exception { // 2 running containers. Resource usedResources = Resources.multiply(containerResource, 2); Resource nmResource = - Resource.newInstance(nm1.getMemory(), nm1.getvCores()); + Resource.newInstance(nm1.getMemory(), nm1.getvCores(), nm1.getVDisks()); assertTrue(schedulerNode1.isValidContainer(amContainer.getContainerId())); assertTrue(schedulerNode1.isValidContainer(runningContainer @@ -348,7 +348,7 @@ private void checkFSQueue(ResourceManager rm, Resource availableResources) throws Exception { // waiting for RM's scheduling apps int retry = 0; - Resource assumedFairShare = Resource.newInstance(8192, 8); + Resource assumedFairShare = Resource.newInstance(8192, 8, 20); while (true) { Thread.sleep(100); if (assumedFairShare.equals(((FairScheduler)rm.getResourceScheduler()) @@ -525,7 +525,7 @@ public void testCapacitySchedulerRecovery() throws Exception { // Calculate each queue's resource usage. Resource containerResource = Resource.newInstance(1024, 1); Resource nmResource = - Resource.newInstance(nm1.getMemory(), nm1.getvCores()); + Resource.newInstance(nm1.getMemory(), nm1.getvCores(), nm1.getVDisks()); Resource clusterResource = Resources.multiply(nmResource, 2); Resource q1Resource = Resources.multiply(clusterResource, 0.5); Resource q2Resource = Resources.multiply(clusterResource, 0.5); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceWeights.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceWeights.java index f420b9e..f14372c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceWeights.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceWeights.java @@ -30,19 +30,25 @@ public void testWeights() { rw1.getWeight(ResourceType.CPU), 0.00001f); Assert.assertEquals("Default memory weight should be 0.0f", 0.0f, rw1.getWeight(ResourceType.MEMORY), 0.00001f); + Assert.assertEquals("Default Disk I/O weight should be 0.0f", 0.0f, + rw1.getWeight(ResourceType.DISKIO), 0.00001f); ResourceWeights rw2 = new ResourceWeights(2.0f); Assert.assertEquals("The CPU weight should be 2.0f.", 2.0f, rw2.getWeight(ResourceType.CPU), 0.00001f); Assert.assertEquals("The memory weight should be 2.0f", 2.0f, rw2.getWeight(ResourceType.MEMORY), 0.00001f); + Assert.assertEquals("Default Disk I/O weight should be 2.0f", 2.0f, + rw2.getWeight(ResourceType.DISKIO), 0.00001f); // set each individually - ResourceWeights rw3 = new ResourceWeights(1.5f, 2.0f); + ResourceWeights rw3 = new ResourceWeights(1.5f, 2.0f, 2.5f); Assert.assertEquals("The CPU weight should be 2.0f", 2.0f, rw3.getWeight(ResourceType.CPU), 0.00001f); Assert.assertEquals("The memory weight should be 1.5f", 1.5f, rw3.getWeight(ResourceType.MEMORY), 0.00001f); + Assert.assertEquals("Default Disk I/O weight should be 2.5f", 2.5f, + rw3.getWeight(ResourceType.DISKIO), 0.00001f); // reset weights rw3.setWeight(ResourceType.CPU, 2.5f); @@ -51,5 +57,8 @@ public void testWeights() { rw3.setWeight(ResourceType.MEMORY, 4.0f); Assert.assertEquals("The memory weight should be set to 4.0f.", 4.0f, rw3.getWeight(ResourceType.MEMORY), 0.00001f); + rw3.setWeight(ResourceType.DISKIO, 5.0f); + Assert.assertEquals("Default Disk I/O weight should be 5.0f", 5.0f, + rw3.getWeight(ResourceType.DISKIO), 0.00001f); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java index ae98660..9fc9ccc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java @@ -24,20 +24,20 @@ public class TestResources { @Test(timeout=1000) public void testFitsIn() { - assertTrue(fitsIn(createResource(1, 1), createResource(2, 2))); - assertTrue(fitsIn(createResource(2, 2), createResource(2, 2))); - assertFalse(fitsIn(createResource(2, 2), createResource(1, 1))); - assertFalse(fitsIn(createResource(1, 2), createResource(2, 1))); - assertFalse(fitsIn(createResource(2, 1), createResource(1, 2))); + assertTrue(fitsIn(createResource(1, 1, 1), createResource(2, 2, 2))); + assertTrue(fitsIn(createResource(2, 2, 2), createResource(2, 2, 2))); + assertFalse(fitsIn(createResource(2, 2, 2), createResource(1, 1, 1))); + assertFalse(fitsIn(createResource(1, 2, 3), createResource(3, 2, 1))); + assertFalse(fitsIn(createResource(2, 1, 3), createResource(1, 2, 3))); } @Test(timeout=1000) public void testComponentwiseMin() { - assertEquals(createResource(1, 1), - componentwiseMin(createResource(1, 1), createResource(2, 2))); - assertEquals(createResource(1, 1), - componentwiseMin(createResource(2, 2), createResource(1, 1))); - assertEquals(createResource(1, 1), - componentwiseMin(createResource(1, 2), createResource(2, 1))); + assertEquals(createResource(1, 1, 1), + componentwiseMin(createResource(1, 1, 1), createResource(2, 2, 2))); + assertEquals(createResource(1, 1, 1), + componentwiseMin(createResource(2, 2, 2), createResource(1, 1, 1))); + assertEquals(createResource(1, 1, 1), + componentwiseMin(createResource(1, 2, 1), createResource(2, 1, 2))); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index c837450..822dd14 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -132,7 +132,7 @@ public void testNMExpiry() throws Exception { String hostname1 = "localhost1"; String hostname2 = "localhost2"; String hostname3 = "localhost3"; - Resource capability = BuilderUtils.newResource(1024, 1); + Resource capability = Resource.newInstance(1024, 1, 1); RegisterNodeManagerRequest request1 = recordFactory .newRecordInstance(RegisterNodeManagerRequest.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java index d16d551..dd9d99a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java @@ -102,7 +102,7 @@ public void setUp() { @Test public void testReconnect() throws Exception { String hostname1 = "localhost1"; - Resource capability = BuilderUtils.newResource(1024, 1); + Resource capability = Resource.newInstance(1024, 1, 1); RegisterNodeManagerRequest request1 = recordFactory .newRecordInstance(RegisterNodeManagerRequest.class); @@ -121,7 +121,7 @@ public void testReconnect() throws Exception { rmNodeEvents.clear(); resourceTrackerService.registerNodeManager(request1); - capability = BuilderUtils.newResource(1024, 2); + capability = Resource.newInstance(1024, 2, 2); request1.setResource(capability); Assert.assertEquals(RMNodeEventType.RECONNECTED, rmNodeEvents.get(0).getType()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java index 4f94695..5e1ec78 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java @@ -94,7 +94,7 @@ public void tearDown() { @Test public void testRPCResponseId() throws IOException, YarnException { String node = "localhost"; - Resource capability = BuilderUtils.newResource(1024, 1); + Resource capability = Resource.newInstance(1024, 1, 1); RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class); nodeId = NodeId.newInstance(node, 1234); request.setNodeId(nodeId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index c9e81ee..d0b7ece 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -91,12 +91,12 @@ @Test (timeout = 30000) public void testNormalizeRequest() { ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); final int minMemory = 1024; final int maxMemory = 8192; - Resource minResource = Resources.createResource(minMemory, 0); - Resource maxResource = Resources.createResource(maxMemory, 0); - + Resource minResource = Resources.createResource(minMemory, 0, 0); + Resource maxResource = Resources.createResource(maxMemory, 0, 0); + ResourceRequest ask = new ResourceRequestPBImpl(); // case negative memory @@ -155,33 +155,35 @@ public void testNormalizeRequest() { public void testNormalizeRequestWithDominantResourceCalculator() { ResourceCalculator resourceCalculator = new DominantResourceCalculator(); - Resource minResource = Resources.createResource(1024, 1); - Resource maxResource = Resources.createResource(10240, 10); - Resource clusterResource = Resources.createResource(10 * 1024, 10); + Resource minResource = Resources.createResource(1024, 1, 1); + Resource maxResource = Resources.createResource(10240, 10, 8); + Resource clusterResource = Resources.createResource(10 * 1024, 10, 8); ResourceRequest ask = new ResourceRequestPBImpl(); - // case negative memory/vcores - ask.setCapability(Resources.createResource(-1024, -1)); + // case negative memory/vcores/vdisks + ask.setCapability(Resources.createResource(-1024, -1, -1)); SchedulerUtils.normalizeRequest( ask, resourceCalculator, clusterResource, minResource, maxResource); assertEquals(minResource, ask.getCapability()); - // case zero memory/vcores - ask.setCapability(Resources.createResource(0, 0)); + // case zero memory/vcores/vdisks + ask.setCapability(Resources.createResource(0, 0, 0)); SchedulerUtils.normalizeRequest( ask, resourceCalculator, clusterResource, minResource, maxResource); assertEquals(minResource, ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(1024, ask.getCapability().getMemory()); + assertEquals(1, ask.getCapability().getVirtualDisks()); - // case non-zero memory & zero cores - ask.setCapability(Resources.createResource(1536, 0)); + // case non-zero memory & zero cores & zero vdisks + ask.setCapability(Resources.createResource(1536, 0, 0)); SchedulerUtils.normalizeRequest( ask, resourceCalculator, clusterResource, minResource, maxResource); - assertEquals(Resources.createResource(2048, 1), ask.getCapability()); + assertEquals(Resources.createResource(2048, 1, 1), ask.getCapability()); assertEquals(1, ask.getCapability().getVirtualCores()); assertEquals(2048, ask.getCapability().getMemory()); + assertEquals(1, ask.getCapability().getVirtualDisks()); } @Test (timeout = 30000) @@ -347,13 +349,15 @@ public void testValidateResourceRequest() { Resource maxResource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_DISK_VDISKS); // zero memory try { Resource resource = Resources.createResource(0, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS); ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -367,7 +371,8 @@ public void testValidateResourceRequest() { try { Resource resource = Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS); ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -377,12 +382,27 @@ public void testValidateResourceRequest() { fail("Zero vcores should be accepted"); } + // zero vdisks + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + 0); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource, null, + mockScheduler); + } catch (InvalidResourceRequestException e) { + fail("Zero vdisks should be accepted"); + } + // max memory try { Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS); ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -397,7 +417,8 @@ public void testValidateResourceRequest() { Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS); ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -407,11 +428,26 @@ public void testValidateResourceRequest() { fail("Max vcores should not be accepted"); } + // max vdisks + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_DISK_VDISKS); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource, null, + mockScheduler); + } catch (InvalidResourceRequestException e) { + fail("Max vdisks should not be accepted"); + } + // negative memory try { Resource resource = Resources.createResource(-1, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS); ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -426,7 +462,8 @@ public void testValidateResourceRequest() { try { Resource resource = Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1); + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS); ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -437,12 +474,28 @@ public void testValidateResourceRequest() { // expected } + // negative vdisks + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + -1); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource, null, + mockScheduler); + fail("Negative vdisks should not be accepted"); + } catch (InvalidResourceRequestException e) { + // expected + } + // more than max memory try { Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS); ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -459,7 +512,8 @@ public void testValidateResourceRequest() { Resources .createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1); + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS); ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1); @@ -469,8 +523,23 @@ public void testValidateResourceRequest() { } catch (InvalidResourceRequestException e) { // expected } + + // more than max vcores + try { + Resource resource = Resources.createResource( + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_DISK_VDISKS + 1); + ResourceRequest resReq = BuilderUtils.newResourceRequest( + mock(Priority.class), ResourceRequest.ANY, resource, 1); + SchedulerUtils.validateResourceRequest(resReq, maxResource, null, + mockScheduler); + fail("More than max vdisks should not be accepted"); + } catch (InvalidResourceRequestException e) { + // expected + } } - + @Test public void testValidateResourceBlacklistRequest() throws Exception { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 2aa57a0..bd30ffe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -383,19 +383,76 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { LOG.info("Setup top-level queues a and b"); } @Test public void testMaximumCapacitySetup() { float delta = 0.0000001f; CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta); conf.setMaximumCapacity(A, 50.0f); assertEquals(50.0f, conf.getMaximumCapacity(A),delta); conf.setMaximumCapacity(A, -1); assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta); + } + + @Test + public void testNoMoreVDisksOnNode() throws Exception { + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class.getName()); + MockRM rm = new MockRM(conf); + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + // Create two nodes, one with 2 vDisks, and one with 1 vDisks + RMNode node1 = + MockNodes.newNodeInfo(0, Resource.newInstance(4 * GB, 2, 2), 1, "host1"); + RMNode node2 = + MockNodes.newNodeInfo(0, Resource.newInstance(4 * GB, 2, 1), 1, "host2"); + cs.handle(new NodeAddedSchedulerEvent(node1)); + cs.handle(new NodeAddedSchedulerEvent(node2)); + + // Create one application + ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1); + ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( + appId1, 1); + SchedulerEvent addAppEvent1 = + new AppAddedSchedulerEvent(appId1, "default", "user"); + cs.handle(addAppEvent1); + SchedulerEvent addAttemptEvent1 = + new AppAttemptAddedSchedulerEvent(appAttemptId1, false); + cs.handle(addAttemptEvent1); + + // Request four containers + List ask1 = new ArrayList(); + ResourceRequest request1 = ResourceRequest.newInstance( + Priority.newInstance(1), ResourceRequest.ANY, + Resource.newInstance(1024, 1, 1), 4, true); + ask1.add(request1); + cs.allocate(appAttemptId1, ask1, Collections.emptyList(), + null, null); + + // For node1, it has 2 vdisks, thus it can assign two containers + FiCaSchedulerApp app1 = cs.getApplicationAttempt(appAttemptId1); + Assert.assertEquals(0, app1.getLiveContainers().size()); + NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1); + cs.handle(nodeUpdateEvent); + Assert.assertEquals(1, app1.getLiveContainers().size()); + cs.handle(nodeUpdateEvent); + Assert.assertEquals(2, app1.getLiveContainers().size()); + + // For node2, it only has 1 vdisks, thus it can only assign one container + NodeUpdateSchedulerEvent nodeUpdateEvent2 = new NodeUpdateSchedulerEvent(node2); + cs.handle(nodeUpdateEvent2); + Assert.assertEquals(3, app1.getLiveContainers().size()); + cs.handle(nodeUpdateEvent2); + Assert.assertEquals(3, app1.getLiveContainers().size()); + + rm.stop(); } - - + @Test public void testRefreshQueues() throws Exception { CapacityScheduler cs = new CapacityScheduler(); @@ -1035,8 +1092,8 @@ public void testPreemptionInfo() throws Exception { // check values waitForAppPreemptionInfo(app0, - Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3, - Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3); + Resource.newInstance(CONTAINER_MEMORY * 3, 3, 3), 0, 3, + Resource.newInstance(CONTAINER_MEMORY * 3, 3, 3), false, 3); // kill app0-attempt0 AM container cs.killContainer(schedulerAppAttempt.getRMContainer(app0 @@ -1047,7 +1104,7 @@ public void testPreemptionInfo() throws Exception { // check values waitForAppPreemptionInfo(app0, - Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3, + Resource.newInstance(CONTAINER_MEMORY * 4, 4, 4), 1, 3, Resource.newInstance(0, 0), false, 0); // launch app0-attempt1 @@ -1065,12 +1122,12 @@ public void testPreemptionInfo() throws Exception { // check values waitForAppPreemptionInfo(app0, - Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6, - Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3); + Resource.newInstance(CONTAINER_MEMORY * 7, 7, 7), 1, 6, + Resource.newInstance(CONTAINER_MEMORY * 3, 3, 3), false, 3); rm1.stop(); } - + @Test(timeout = 30000) public void testRecoverRequestAfterPreemption() throws Exception { Configuration conf = new Configuration(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java index 7b6aaf3..c6bab00 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java @@ -102,8 +102,15 @@ protected ResourceRequest createResourceRequest( protected ResourceRequest createResourceRequest( int memory, int vcores, String host, int priority, int numContainers, boolean relaxLocality) { + return createResourceRequest(memory, vcores, 0, host, priority, + numContainers, relaxLocality); + } + + protected ResourceRequest createResourceRequest( + int memory, int vcores, int vDisks, String host, int priority, + int numContainers, boolean relaxLocality) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); - request.setCapability(BuilderUtils.newResource(memory, vcores)); + request.setCapability(Resource.newInstance(memory, vcores, vDisks)); request.setResourceName(host); request.setNumContainers(numContainers); Priority prio = recordFactory.newRecordInstance(Priority.class); @@ -146,6 +153,13 @@ protected ApplicationAttemptId createSchedulingRequest( protected ApplicationAttemptId createSchedulingRequest( int memory, int vcores, String queueId, String userId, int numContainers, int priority) { + return createSchedulingRequest(memory, vcores, 0, queueId, userId, + numContainers, priority); + } + + protected ApplicationAttemptId createSchedulingRequest( + int memory, int vcores, int vDisks, String queueId, String userId, + int numContainers, int priority) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); scheduler.addApplication(id.getApplicationId(), queueId, userId, false); // This conditional is for testAclSubmitApplication where app is rejected @@ -154,8 +168,8 @@ protected ApplicationAttemptId createSchedulingRequest( scheduler.addApplicationAttempt(id, false, false); } List ask = new ArrayList(); - ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, - priority, numContainers, true); + ResourceRequest request = createResourceRequest(memory, vcores, vDisks, + ResourceRequest.ANY, priority, numContainers, true); ask.add(request); scheduler.allocate(id, ask, new ArrayList(), null, null); RMApp rmApp = mock(RMApp.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 9a66a94..751d9ca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -232,9 +232,9 @@ public void testAllocationFileParsing() throws Exception { assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(Resources.createResource(1024, 0), + assertEquals(Resources.createResource(1024, 0, 0), queueConf.getMinResources("root.queueA")); - assertEquals(Resources.createResource(2048, 0), + assertEquals(Resources.createResource(2048, 0, 0), queueConf.getMinResources("root.queueB")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueC")); @@ -405,9 +405,9 @@ public void testBackwardsCompatibleAllocationFileParsing() throws Exception { assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); - assertEquals(Resources.createResource(1024, 0), + assertEquals(Resources.createResource(1024, 0, 0), queueConf.getMinResources("root.queueA")); - assertEquals(Resources.createResource(2048, 0), + assertEquals(Resources.createResource(2048, 0, 0), queueConf.getMinResources("root.queueB")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueC")); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java index 9d8dd07..d729963 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java @@ -177,19 +177,19 @@ public void testEmptyList() { */ @Test public void testCPU() { - scheds.add(new FakeSchedulable(Resources.createResource(0, 20), + scheds.add(new FakeSchedulable(Resources.createResource(0, 20, 0), new ResourceWeights(2.0f))); - scheds.add(new FakeSchedulable(Resources.createResource(0, 0), + scheds.add(new FakeSchedulable(Resources.createResource(0, 0, 0), new ResourceWeights(1.0f))); - scheds.add(new FakeSchedulable(Resources.createResource(0, 5), + scheds.add(new FakeSchedulable(Resources.createResource(0, 5, 0), new ResourceWeights(1.0f))); - scheds.add(new FakeSchedulable(Resources.createResource(0, 15), + scheds.add(new FakeSchedulable(Resources.createResource(0, 15, 0), new ResourceWeights(0.5f))); ComputeFairShares.computeShares(scheds, - Resources.createResource(0, 45), ResourceType.CPU); + Resources.createResource(0, 45, 0), ResourceType.CPU); verifyCPUShares(20, 5, 5, 15); } - + /** * Check that a given list of shares have been assigned to this.scheds. */ diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 67cea37..00078cc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -197,7 +197,7 @@ public void testLoadConfigurationOnInitialize() throws IOException { 5000); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); - conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, + conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 128); scheduler.init(conf); scheduler.start(); @@ -2822,6 +2822,47 @@ public void testNoMoreCpuOnNode() throws IOException { } @Test + public void testNoMoreVDisksOnNode() throws IOException { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2, 2), + 1, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2, 1), + 1, "127.0.0.2"); + NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); + scheduler.handle(nodeEvent2); + + ApplicationAttemptId attId1 = createSchedulingRequest(1024, 1, 1, "default", + "user1", 2, 1); + FSAppAttempt app1 = scheduler.getSchedulerApp(attId1); + scheduler.update(); + + // For node1, it has enough resources for running two containers + NodeUpdateSchedulerEvent updateEvent1 = new NodeUpdateSchedulerEvent(node1); + scheduler.handle(updateEvent1); + assertEquals(1, app1.getLiveContainers().size()); + scheduler.handle(updateEvent1); + assertEquals(2, app1.getLiveContainers().size()); + + ApplicationAttemptId attId2 = createSchedulingRequest(1024, 1, 1, "default", + "user1", 2, 1); + FSAppAttempt app2 = scheduler.getSchedulerApp(attId2); + scheduler.update(); + + // For node2, it can only run one container as it only has 1 vDisk. + NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2); + scheduler.handle(updateEvent2); + assertEquals(1, app2.getLiveContainers().size()); + scheduler.handle(updateEvent2); + assertEquals(1, app2.getLiveContainers().size()); + } + + @Test public void testBasicDRFAssignment() throws Exception { scheduler.init(conf); scheduler.start(); @@ -3455,6 +3496,7 @@ public void testContinuousScheduling() throws Exception { Configuration conf = createConfiguration(); conf.setBoolean(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, true); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_DISK_VDISKS, 0); scheduler.setRMContext(resourceManager.getRMContext()); scheduler.init(conf); scheduler.start(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java index a5c20c1..eb7e792 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java @@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FakeSchedulable; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Test; @@ -38,126 +37,135 @@ public class TestDominantResourceFairnessPolicy { private Comparator createComparator(int clusterMem, - int clusterCpu) { + int clusterCpu, int clusterVdisk) { + return createComparator( + Resource.newInstance(clusterMem, clusterCpu, clusterVdisk)); + } + + private Comparator createComparator(Resource capacity) { DominantResourceFairnessPolicy policy = new DominantResourceFairnessPolicy(); - policy.initialize(BuilderUtils.newResource(clusterMem, clusterCpu)); + policy.initialize(capacity); return policy.getComparator(); } - - private Schedulable createSchedulable(int memUsage, int cpuUsage) { - return createSchedulable(memUsage, cpuUsage, ResourceWeights.NEUTRAL, 0, 0); + + private Schedulable createSchedulable(int memUsage, int cpuUsage, + int vdiskUsage) { + return createSchedulable(memUsage, cpuUsage, vdiskUsage, + ResourceWeights.NEUTRAL, 0, 0, 0); } - + private Schedulable createSchedulable(int memUsage, int cpuUsage, - int minMemShare, int minCpuShare) { - return createSchedulable(memUsage, cpuUsage, ResourceWeights.NEUTRAL, - minMemShare, minCpuShare); + int vdiskUsage, int minMemShare, int minCpuShare, int minVdiskShare) { + return createSchedulable(memUsage, cpuUsage, vdiskUsage, + ResourceWeights.NEUTRAL, minMemShare, minCpuShare, minVdiskShare); } - private Schedulable createSchedulable(int memUsage, int cpuUsage, - ResourceWeights weights) { - return createSchedulable(memUsage, cpuUsage, weights, 0, 0); + int vdiskUsage, ResourceWeights weights) { + return createSchedulable(memUsage, cpuUsage, vdiskUsage, weights, 0, 0, 0); } - private Schedulable createSchedulable(int memUsage, int cpuUsage, - ResourceWeights weights, int minMemShare, int minCpuShare) { - Resource usage = BuilderUtils.newResource(memUsage, cpuUsage); - Resource minShare = BuilderUtils.newResource(minMemShare, minCpuShare); + int vdiskUsage, ResourceWeights weights, int minMemShare, + int minCpuShare, int minVdiskShare) { + Resource usage = Resource.newInstance(memUsage, cpuUsage, vdiskUsage); + Resource minShare = + Resource.newInstance(minMemShare, minCpuShare, minVdiskShare); return new FakeSchedulable(minShare, Resources.createResource(Integer.MAX_VALUE, Integer.MAX_VALUE), weights, Resources.none(), usage, 0l); } @Test public void testSameDominantResource() { - assertTrue(createComparator(8000, 4).compare( - createSchedulable(1000, 1), - createSchedulable(2000, 1)) < 0); + assertTrue(createComparator(8000, 4, 4).compare( + createSchedulable(1000, 1, 1), + createSchedulable(2000, 1, 1)) < 0); } @Test public void testDifferentDominantResource() { - assertTrue(createComparator(8000, 8).compare( - createSchedulable(4000, 3), - createSchedulable(2000, 5)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(4000, 3, 2), + createSchedulable(2000, 5, 3)) < 0); } @Test public void testOneIsNeedy() { - assertTrue(createComparator(8000, 8).compare( - createSchedulable(2000, 5, 0, 6), - createSchedulable(4000, 3, 0, 0)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(2000, 5, 1, 0, 6, 0), + createSchedulable(4000, 3, 1, 0, 0, 0)) < 0); } @Test public void testBothAreNeedy() { - assertTrue(createComparator(8000, 100).compare( + assertTrue(createComparator(8000, 100, 200).compare( // dominant share is 2000/8000 - createSchedulable(2000, 5), + createSchedulable(2000, 5, 10), // dominant share is 4000/8000 - createSchedulable(4000, 3)) < 0); - assertTrue(createComparator(8000, 100).compare( + createSchedulable(4000, 3, 6)) < 0); + assertTrue(createComparator(8000, 100, 200).compare( // dominant min share is 2/3 - createSchedulable(2000, 5, 3000, 6), + createSchedulable(2000, 5, 5, 3000, 6, 6), // dominant min share is 4/5 - createSchedulable(4000, 3, 5000, 4)) < 0); + createSchedulable(4000, 3, 3, 5000, 4, 4)) < 0); } @Test public void testEvenWeightsSameDominantResource() { - assertTrue(createComparator(8000, 8).compare( - createSchedulable(3000, 1, new ResourceWeights(2.0f)), - createSchedulable(2000, 1)) < 0); - assertTrue(createComparator(8000, 8).compare( - createSchedulable(1000, 3, new ResourceWeights(2.0f)), - createSchedulable(1000, 2)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(3000, 1, 2, new ResourceWeights(2.0f)), + createSchedulable(2000, 1, 2)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(1000, 3, 6, new ResourceWeights(2.0f)), + createSchedulable(1000, 2, 4)) < 0); } - + @Test public void testEvenWeightsDifferentDominantResource() { - assertTrue(createComparator(8000, 8).compare( - createSchedulable(1000, 3, new ResourceWeights(2.0f)), - createSchedulable(2000, 1)) < 0); - assertTrue(createComparator(8000, 8).compare( - createSchedulable(3000, 1, new ResourceWeights(2.0f)), - createSchedulable(1000, 2)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(1000, 3, 6, new ResourceWeights(2.0f)), + createSchedulable(2000, 1, 2)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(3000, 1, 2, new ResourceWeights(2.0f)), + createSchedulable(1000, 2, 4)) < 0); } @Test public void testUnevenWeightsSameDominantResource() { - assertTrue(createComparator(8000, 8).compare( - createSchedulable(3000, 1, new ResourceWeights(2.0f, 1.0f)), - createSchedulable(2000, 1)) < 0); - assertTrue(createComparator(8000, 8).compare( - createSchedulable(1000, 3, new ResourceWeights(1.0f, 2.0f)), - createSchedulable(1000, 2)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(3000, 1, 2, new ResourceWeights(3.0f, 2.0f, 1.0f)), + createSchedulable(2000, 1, 2)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(1000, 3, 6, new ResourceWeights(1.0f, 2.0f, 3.0f)), + createSchedulable(1000, 2, 4)) < 0); } @Test public void testUnevenWeightsDifferentDominantResource() { - assertTrue(createComparator(8000, 8).compare( - createSchedulable(1000, 3, new ResourceWeights(1.0f, 2.0f)), - createSchedulable(2000, 1)) < 0); - assertTrue(createComparator(8000, 8).compare( - createSchedulable(3000, 1, new ResourceWeights(2.0f, 1.0f)), - createSchedulable(1000, 2)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(1000, 3, 6, new ResourceWeights(1.0f, 2.0f, 3.0f)), + createSchedulable(2000, 1, 2)) < 0); + assertTrue(createComparator(8000, 8, 20).compare( + createSchedulable(3000, 1, 2, new ResourceWeights(3.0f, 2.0f, 1.0f)), + createSchedulable(1000, 2, 4)) < 0); } @Test public void testCalculateShares() { - Resource used = Resources.createResource(10, 5); - Resource capacity = Resources.createResource(100, 10); - ResourceType[] resourceOrder = new ResourceType[2]; + Resource used = Resources.createResource(10, 5, 8); + Resource capacity = Resources.createResource(100, 10, 20); + ResourceType[] resourceOrder = new ResourceType[3]; ResourceWeights shares = new ResourceWeights(); DominantResourceFairnessPolicy.DominantResourceFairnessComparator comparator = new DominantResourceFairnessPolicy.DominantResourceFairnessComparator(); comparator.calculateShares(used, capacity, shares, resourceOrder, ResourceWeights.NEUTRAL); assertEquals(.1, shares.getWeight(ResourceType.MEMORY), .00001); assertEquals(.5, shares.getWeight(ResourceType.CPU), .00001); + assertEquals(.4, shares.getWeight(ResourceType.DISKIO), .00001); assertEquals(ResourceType.CPU, resourceOrder[0]); - assertEquals(ResourceType.MEMORY, resourceOrder[1]); + assertEquals(ResourceType.DISKIO, resourceOrder[1]); + assertEquals(ResourceType.MEMORY, resourceOrder[2]); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 87bacc6..2219aa5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -591,10 +591,7 @@ public void testPerUserResourcesJSON() throws Exception { @Test public void testResourceInfo() { - Resource res = Resources.createResource(10, 1); - // If we add a new resource (e.g disks), then - // CapacitySchedulerPage and these RM WebServices + docs need to be updated - // eg. ResourceInfo - assertEquals("", res.toString()); + Resource res = Resources.createResource(10, 1, 1); + assertEquals("", res.toString()); } }