diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java index beaafe1..e56ca8b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java @@ -43,6 +43,9 @@ @Metric MutableGaugeInt availableGB; @Metric("Current allocated Virtual Cores") MutableGaugeInt allocatedVCores; + @Metric("Current allocated Virtual Disks") + MutableGaugeInt allocatedVDisks; + @Metric MutableGaugeInt availableVDisks; @Metric MutableGaugeInt availableVCores; @Metric("Container launch duration") MutableRate containerLaunchDuration; @@ -101,6 +104,8 @@ public void allocateContainer(Resource res) { availableGB.set((int)Math.floor(availableMB/1024d)); allocatedVCores.incr(res.getVirtualCores()); availableVCores.decr(res.getVirtualCores()); + allocatedVDisks.incr(res.getVirtualDisks()); + availableVDisks.decr(res.getVirtualDisks()); } public void releaseContainer(Resource res) { @@ -111,12 +116,15 @@ public void releaseContainer(Resource res) { availableGB.set((int)Math.floor(availableMB/1024d)); allocatedVCores.decr(res.getVirtualCores()); availableVCores.incr(res.getVirtualCores()); + allocatedVDisks.decr(res.getVirtualDisks()); + availableVDisks.incr(res.getVirtualDisks()); } public void addResource(Resource res) { availableMB = availableMB + res.getMemory(); availableGB.incr((int)Math.floor(availableMB/1024d)); availableVCores.incr(res.getVirtualCores()); + availableVDisks.incr(res.getVirtualDisks()); } public void addContainerLaunchDuration(long value) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java index d2a0691..f90a91d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java @@ -32,10 +32,11 @@ Resource total = Records.newRecord(Resource.class); total.setMemory(8*GiB); total.setVirtualCores(16); + total.setVirtualDisks(20); Resource resource = Records.newRecord(Resource.class); resource.setMemory(512); //512MiB resource.setVirtualCores(2); - + resource.setVirtualDisks(2); metrics.addResource(total); @@ -67,13 +68,14 @@ // while allocatedGB is expected to be ceiled. // allocatedGB: 3.5GB allocated memory is shown as 4GB // availableGB: 4.5GB available memory is shown as 4GB - checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2); + checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2, 14, 6); } private void checkMetrics(int launched, int completed, int failed, int killed, int initing, int running, int allocatedGB, - int allocatedContainers, int availableGB, int allocatedVCores, - int availableVCores) { + int allocatedContainers, int availableGB, + int allocatedVCores, int availableVCores, + int allocatedVDisks, int availableVDisks) { MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics"); assertCounter("ContainersLaunched", launched, rb); assertCounter("ContainersCompleted", completed, rb); @@ -83,9 +85,10 @@ private void checkMetrics(int launched, int completed, int failed, int killed, assertGauge("ContainersRunning", running, rb); assertGauge("AllocatedGB", allocatedGB, rb); assertGauge("AllocatedVCores", allocatedVCores, rb); + assertGauge("AllocatedVDisks", allocatedVDisks, rb); assertGauge("AllocatedContainers", allocatedContainers, rb); assertGauge("AvailableGB", availableGB, rb); assertGauge("AvailableVCores",availableVCores, rb); - + assertGauge("AvailableVDisks", availableVDisks, rb); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 507b798..2358974 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.resource.Resources; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,16 +60,20 @@ @Metric("Allocated memory in MB") MutableGaugeInt allocatedMB; @Metric("Allocated CPU in virtual cores") MutableGaugeInt allocatedVCores; + @Metric("Allocated disk I/O in virtual disks") MutableGaugeInt allocatedVDisks; @Metric("# of allocated containers") MutableGaugeInt allocatedContainers; @Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; @Metric("Available memory in MB") MutableGaugeInt availableMB; @Metric("Available CPU in virtual cores") MutableGaugeInt availableVCores; + @Metric("Available disk I/O in virtual disks") MutableGaugeInt availableVDisks; @Metric("Pending memory allocation in MB") MutableGaugeInt pendingMB; @Metric("Pending CPU allocation in virtual cores") MutableGaugeInt pendingVCores; + @Metric("Pending disk I/O allocation in virtual disks") MutableGaugeInt pendingVDisks; @Metric("# of pending containers") MutableGaugeInt pendingContainers; @Metric("# of reserved memory in MB") MutableGaugeInt reservedMB; @Metric("Reserved CPU in virtual cores") MutableGaugeInt reservedVCores; + @Metric("Reserved disk I/O in virtual disks") MutableGaugeInt reservedVDisks; @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active applications") MutableGaugeInt activeApplications; @@ -319,6 +322,7 @@ public void moveAppTo(AppSchedulingInfo app) { public void setAvailableResourcesToQueue(Resource limit) { availableMB.set(limit.getMemory()); availableVCores.set(limit.getVirtualCores()); + availableVDisks.set(limit.getVirtualDisks()); } /** @@ -356,6 +360,7 @@ private void _incrPendingResources(int containers, Resource res) { pendingContainers.incr(containers); pendingMB.incr(res.getMemory() * containers); pendingVCores.incr(res.getVirtualCores() * containers); + pendingVDisks.incr(res.getVirtualDisks() * containers); } public void decrPendingResources(String user, int containers, Resource res) { @@ -373,6 +378,7 @@ private void _decrPendingResources(int containers, Resource res) { pendingContainers.decr(containers); pendingMB.decr(res.getMemory() * containers); pendingVCores.decr(res.getVirtualCores() * containers); + pendingVDisks.decr(res.getVirtualDisks() * containers); } public void allocateResources(String user, int containers, Resource res, @@ -381,6 +387,7 @@ public void allocateResources(String user, int containers, Resource res, aggregateContainersAllocated.incr(containers); allocatedMB.incr(res.getMemory() * containers); allocatedVCores.incr(res.getVirtualCores() * containers); + allocatedVDisks.incr(res.getVirtualDisks() * containers); if (decrPending) { _decrPendingResources(containers, res); } @@ -398,6 +405,7 @@ public void releaseResources(String user, int containers, Resource res) { aggregateContainersReleased.incr(containers); allocatedMB.decr(res.getMemory() * containers); allocatedVCores.decr(res.getVirtualCores() * containers); + allocatedVDisks.decr(res.getVirtualDisks() * containers); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.releaseResources(user, containers, res); @@ -411,6 +419,7 @@ public void reserveResource(String user, Resource res) { reservedContainers.incr(); reservedMB.incr(res.getMemory()); reservedVCores.incr(res.getVirtualCores()); + reservedVDisks.incr(res.getVirtualDisks()); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.reserveResource(user, res); @@ -424,6 +433,7 @@ public void unreserveResource(String user, Resource res) { reservedContainers.decr(); reservedMB.decr(res.getMemory()); reservedVCores.decr(res.getVirtualCores()); + reservedVDisks.decr(res.getVirtualDisks()); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.unreserveResource(user, res); @@ -488,7 +498,8 @@ public int getAppsFailed() { } public Resource getAllocatedResources() { - return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value()); + return Resource.newInstance(allocatedMB.value(), + allocatedVCores.value(), allocatedVDisks.value()); } public int getAllocatedMB() { @@ -499,6 +510,10 @@ public int getAllocatedVirtualCores() { return allocatedVCores.value(); } + public int getAllocatedVirtualDisks() { + return allocatedVDisks.value(); + } + public int getAllocatedContainers() { return allocatedContainers.value(); } @@ -511,6 +526,10 @@ public int getAvailableVirtualCores() { return availableVCores.value(); } + public int getAvailableVirtualDisks() { + return availableVDisks.value(); + } + public int getPendingMB() { return pendingMB.value(); } @@ -519,6 +538,10 @@ public int getPendingVirtualCores() { return pendingVCores.value(); } + public int getPendingVirtualDisks() { + return pendingVDisks.value(); + } + public int getPendingContainers() { return pendingContainers.value(); } @@ -531,6 +554,10 @@ public int getReservedVirtualCores() { return reservedVCores.value(); } + public int getReservedVirtualDisks() { + return reservedVDisks.value(); + } + public int getReservedContainers() { return reservedContainers.value(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java index 82c422b..a2cd6ba 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java @@ -33,12 +33,16 @@ @Metric("Fair share of memory in MB") MutableGaugeInt fairShareMB; @Metric("Fair share of CPU in vcores") MutableGaugeInt fairShareVCores; + @Metric("Fair share of disk IO in vdisks") MutableGaugeInt fairShareVDisks; @Metric("Steady fair share of memory in MB") MutableGaugeInt steadyFairShareMB; @Metric("Steady fair share of CPU in vcores") MutableGaugeInt steadyFairShareVCores; + @Metric("Steady fair share of disk IO in vdisks") MutableGaugeInt steadyFairShareVDisks; @Metric("Minimum share of memory in MB") MutableGaugeInt minShareMB; @Metric("Minimum share of CPU in vcores") MutableGaugeInt minShareVCores; + @Metric("Minimum share of disk IO in vdisks") MutableGaugeInt minShareVDisks; @Metric("Maximum share of memory in MB") MutableGaugeInt maxShareMB; @Metric("Maximum share of CPU in vcores") MutableGaugeInt maxShareVCores; + @Metric("Maximum share of disk IO in vdisks") MutableGaugeInt maxShareVDisks; FSQueueMetrics(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { @@ -48,6 +52,7 @@ public void setFairShare(Resource resource) { fairShareMB.set(resource.getMemory()); fairShareVCores.set(resource.getVirtualCores()); + fairShareVDisks.set(resource.getVirtualDisks()); } public int getFairShareMB() { @@ -61,6 +66,7 @@ public int getFairShareVirtualCores() { public void setSteadyFairShare(Resource resource) { steadyFairShareMB.set(resource.getMemory()); steadyFairShareVCores.set(resource.getVirtualCores()); + steadyFairShareVDisks.set(resource.getVirtualDisks()); } public int getSteadyFairShareMB() { @@ -71,9 +77,14 @@ public int getSteadyFairShareVCores() { return steadyFairShareVCores.value(); } + public int getSteadyFairShareVDisks() { + return steadyFairShareVDisks.value(); + } + public void setMinShare(Resource resource) { minShareMB.set(resource.getMemory()); minShareVCores.set(resource.getVirtualCores()); + minShareVDisks.set(resource.getVirtualDisks()); } public int getMinShareMB() { @@ -83,10 +94,15 @@ public int getMinShareMB() { public int getMinShareVirtualCores() { return minShareVCores.value(); } - + + public int getMinShareVirtualDisks() { + return minShareVDisks.value(); + } + public void setMaxShare(Resource resource) { maxShareMB.set(resource.getMemory()); maxShareVCores.set(resource.getVirtualCores()); + maxShareVDisks.set(resource.getVirtualDisks()); } public int getMaxShareMB() { @@ -96,7 +112,11 @@ public int getMaxShareMB() { public int getMaxShareVirtualCores() { return maxShareVCores.value(); } - + + public int getMaxShareVirtualDisks() { + return maxShareVDisks.value(); + } + public synchronized static FSQueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java index 88efe47..c2f0b47 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java @@ -72,6 +72,9 @@ protected void render(Block html) { th().$class("ui-state-default")._("VCores Used")._(). th().$class("ui-state-default")._("VCores Total")._(). th().$class("ui-state-default")._("VCores Reserved")._(). + th().$class("ui-state-default")._("VDisks Used")._(). + th().$class("ui-state-default")._("VDisks Total")._(). + th().$class("ui-state-default")._("VDisks Reserved")._(). th().$class("ui-state-default")._("Active Nodes")._(). th().$class("ui-state-default")._("Decommissioned Nodes")._(). th().$class("ui-state-default")._("Lost Nodes")._(). @@ -97,6 +100,9 @@ protected void render(Block html) { td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())). td(String.valueOf(clusterMetrics.getTotalVirtualCores())). td(String.valueOf(clusterMetrics.getReservedVirtualCores())). + td(String.valueOf(clusterMetrics.getAllocatedVirtualDisks())). + td(String.valueOf(clusterMetrics.getTotalVirtualCores())). + td(String.valueOf(clusterMetrics.getReservedVirtualDisks())). td().a(url("nodes"),String.valueOf(clusterMetrics.getActiveNodes()))._(). td().a(url("nodes/decommissioned"),String.valueOf(clusterMetrics.getDecommissionedNodes()))._(). td().a(url("nodes/lost"),String.valueOf(clusterMetrics.getLostNodes()))._(). @@ -126,6 +132,9 @@ protected void render(Block html) { th().$class("ui-state-default")._("VCores Used")._(). th().$class("ui-state-default")._("VCores Pending")._(). th().$class("ui-state-default")._("VCores Reserved")._(). + th().$class("ui-state-default")._("VDisks Used")._(). + th().$class("ui-state-default")._("VDisks Pending")._(). + th().$class("ui-state-default")._("VDisks Reserved")._(). _(). _(). tbody().$class("ui-widget-content"). @@ -148,6 +157,9 @@ protected void render(Block html) { td(String.valueOf(userMetrics.getAllocatedVirtualCores())). td(String.valueOf(userMetrics.getPendingVirtualCores())). td(String.valueOf(userMetrics.getReservedVirtualCores())). + td(String.valueOf(userMetrics.getAllocatedVirtualDisks())). + td(String.valueOf(userMetrics.getPendingVirtualDisks())). + td(String.valueOf(userMetrics.getReservedVirtualDisks())). _(). _()._(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index 16a5c01..cba90d2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -45,6 +45,10 @@ protected long availableVirtualCores; protected long allocatedVirtualCores; + protected long reservedVirtualDisks; + protected long availableVirtualDisks; + protected long allocatedVirtualDisks; + protected int containersAllocated; protected int containersReserved; protected int containersPending; @@ -81,6 +85,10 @@ public ClusterMetricsInfo(final ResourceManager rm) { this.availableVirtualCores = metrics.getAvailableVirtualCores(); this.allocatedVirtualCores = metrics.getAllocatedVirtualCores(); + this.reservedVirtualDisks = metrics.getReservedVirtualDisks(); + this.availableVirtualDisks = metrics.getAvailableVirtualDisks(); + this.allocatedVirtualDisks = metrics.getAllocatedVirtualDisks(); + this.containersAllocated = metrics.getAllocatedContainers(); this.containersPending = metrics.getPendingContainers(); this.containersReserved = metrics.getReservedContainers(); @@ -144,6 +152,18 @@ public long getAllocatedVirtualCores() { return this.allocatedVirtualCores; } + public long getReservedVirtualDisks() { + return this.reservedVirtualDisks; + } + + public long getAvailableVirtualDisks() { + return this.reservedVirtualDisks; + } + + public long getAllocatedVirtualDisks() { + return this.allocatedVirtualDisks; + } + public int getContainersAllocated() { return this.containersAllocated; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java index 73a83d7..a4e636c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java @@ -46,6 +46,9 @@ protected long reservedVirtualCores; protected long pendingVirtualCores; protected long allocatedVirtualCores; + protected long reservedVirtualDisks; + protected long pendingVirtualDisks; + protected long allocatedVirtualDisks; @XmlTransient protected boolean userMetricsAvailable; @@ -80,6 +83,10 @@ public UserMetricsInfo(final ResourceManager rm, final String user) { this.reservedVirtualCores = userMetrics.getReservedVirtualCores(); this.pendingVirtualCores = userMetrics.getPendingVirtualCores(); this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores(); + + this.reservedVirtualDisks = userMetrics.getReservedVirtualDisks(); + this.pendingVirtualDisks = userMetrics.getPendingVirtualDisks(); + this.allocatedVirtualDisks = userMetrics.getAllocatedVirtualDisks(); } } @@ -135,6 +142,18 @@ public long getPendingVirtualCores() { return this.pendingVirtualCores; } + public long getReservedVirtualDisks() { + return this.reservedVirtualDisks; + } + + public long getAllocatedVirtualDisks() { + return this.allocatedVirtualDisks; + } + + public long getPendingVirtualDisks() { + return this.pendingVirtualDisks; + } + public int getReservedContainers() { return this.reservedContainers; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 8ad71d2..05b4c72 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -72,20 +72,23 @@ public void setUp() { metrics.submitAppAttempt(user); checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); - metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100)); - metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3)); + metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100, 100)); + metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources - checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); + checkResources(queueSource, 0, 0, 0, 0, 0, 0, 100*GB, 100, 100, 15*GB, 15, + 15, 5, 0, 0, 0, 0); metrics.runAppAttempt(app.getApplicationId(), user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); - metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true); - checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); + metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2, 2), true); + checkResources(queueSource, 6*GB, 6, 6, 3, 3, 0, 100*GB, 100, 100, 9*GB, 9, + 9, 2, 0, 0, 0, 0); - metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2)); - checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); + metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2, 2)); + checkResources(queueSource, 4*GB, 4, 4, 2, 3, 1, 100*GB, 100, 100, 9*GB, 9, + 9, 2, 0, 0, 0, 0); metrics.finishAppAttempt( app.getApplicationId(), app.isPending(), app.getUser()); @@ -169,25 +172,31 @@ public void testQueueAppMetricsForMultipleFailures() { checkApps(queueSource, 1, 1, 0, 0, 0, 0, true); checkApps(userSource, 1, 1, 0, 0, 0, 0, true); - metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100)); - metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10)); - metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3)); + metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100, 100)); + metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10, 10)); + metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources - checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); - checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0); + checkResources(queueSource, 0, 0, 0, 0, 0, 0, 100*GB, 100, 100, 15*GB, 15, + 15, 5, 0, 0, 0, 0); + checkResources(userSource, 0, 0, 0, 0, 0, 0, 10*GB, 10, 10, 15*GB, 15, 15, + 5, 0, 0, 0, 0); metrics.runAppAttempt(app.getApplicationId(), user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); checkApps(userSource, 1, 0, 1, 0, 0, 0, true); - metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true); - checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); - checkResources(userSource, 6*GB, 6, 3, 3, 0, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0); + metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2, 2), true); + checkResources(queueSource, 6*GB, 6, 6, 3, 3, 0, 100*GB, 100, 100, 9*GB, 9, + 9, 2, 0, 0, 0, 0); + checkResources(userSource, 6*GB, 6, 6, 3, 3, 0, 10*GB, 10, 10, 9*GB, 9, 9, + 2, 0, 0, 0, 0); - metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2)); - checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); - checkResources(userSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0); + metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2, 2)); + checkResources(queueSource, 4*GB, 4, 4, 2, 3, 1, 100*GB, 100, 100, 9*GB, 9, + 9, 2, 0, 0, 0, 0); + checkResources(userSource, 4*GB, 4, 4, 2, 3, 1, 10*GB, 10, 10, 9*GB, 9, 9, + 2, 0, 0, 0, 0); metrics.finishAppAttempt( app.getApplicationId(), app.isPending(), app.getUser()); @@ -228,35 +237,47 @@ public void testQueueAppMetricsForMultipleFailures() { checkApps(userSource, 1, 1, 0, 0, 0, 0, true); checkApps(parentUserSource, 1, 1, 0, 0, 0, 0, true); - parentMetrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100)); - metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100)); - parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10)); - metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10)); - metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3)); - checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); - checkResources(parentQueueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0); - checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0); - checkResources(parentUserSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0); + parentMetrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100, 100)); + metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100, 100)); + parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10, 10)); + metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10, 10)); + metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3, 3)); + checkResources(queueSource, 0, 0, 0, 0, 0, 0, 100*GB, 100, 100, 15*GB, 15, + 15, 5, 0, 0, 0, 0); + checkResources(parentQueueSource, 0, 0, 0, 0, 0, 0, 100*GB, 100, 100, 15*GB, + 15, 15, 5, 0, 0, 0, 0); + checkResources(userSource, 0, 0, 0, 0, 0, 0, 10*GB, 10, 10, 15*GB, 15, 15, + 5, 0, 0, 0, 0); + checkResources(parentUserSource, 0, 0, 0, 0, 0, 0, 10*GB, 10, 10, 15*GB, 15, + 15, 5, 0, 0, 0, 0); metrics.runAppAttempt(app.getApplicationId(), user); checkApps(queueSource, 1, 0, 1, 0, 0, 0, true); checkApps(userSource, 1, 0, 1, 0, 0, 0, true); - metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true); - metrics.reserveResource(user, Resources.createResource(3*GB, 3)); + metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2, 2), true); + metrics.reserveResource(user, Resources.createResource(3*GB, 3, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources - checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 3*GB, 3, 1); - checkResources(parentQueueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 3*GB, 3, 1); - checkResources(userSource, 6*GB, 6, 3, 3, 0, 10*GB, 10, 9*GB, 9, 2, 3*GB, 3, 1); - checkResources(parentUserSource, 6*GB, 6, 3, 3, 0, 10*GB, 10, 9*GB, 9, 2, 3*GB, 3, 1); - - metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2)); - metrics.unreserveResource(user, Resources.createResource(3*GB, 3)); - checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); - checkResources(parentQueueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0); - checkResources(userSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0); - checkResources(parentUserSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0); + checkResources(queueSource, 6*GB, 6, 6, 3, 3, 0, 100*GB, 100, 100, 9*GB, 9, + 9, 2, 3*GB, 3, 3, 1); + checkResources(parentQueueSource, 6*GB, 6, 6, 3, 3, 0, 100*GB, 100, 100, + 9*GB, 9, 9, 2, 3*GB, 3, 3, 1); + checkResources(userSource, 6*GB, 6, 6, 3, 3, 0, 10*GB, 10, 10, 9*GB, 9, 9, + 2, 3*GB, 3, 3, 1); + checkResources(parentUserSource, 6*GB, 6, 6, 3, 3, 0, 10*GB, 10, 10, 9*GB, + 9, 9, 2, 3*GB, 3, 3, 1); + + metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2, 2)); + metrics.unreserveResource(user, Resources.createResource(3*GB, 3, 3)); + checkResources(queueSource, 4*GB, 4, 4, 2, 3, 1, 100*GB, 100, 100, 9*GB, 9, + 9, 2, 0, 0, 0, 0); + checkResources(parentQueueSource, 4*GB, 4, 4, 2, 3, 1, 100*GB, 100, 100, + 9*GB, 9, 9, 2, 0, 0, 0, 0); + checkResources(userSource, 4*GB, 4, 4, 2, 3, 1, 10*GB, 10, 10, 9*GB, 9, 9, + 2, 0, 0, 0, 0); + checkResources(parentUserSource, 4*GB, 4, 4, 2, 3, 1, 10*GB, 10, 10, 9*GB, + 9, 9, 2, 0, 0, 0, 0); metrics.finishAppAttempt( app.getApplicationId(), app.isPending(), app.getUser()); @@ -347,23 +368,28 @@ public static void checkApps(MetricsSource source, int submitted, int pending, } public static void checkResources(MetricsSource source, int allocatedMB, - int allocatedCores, int allocCtnrs, long aggreAllocCtnrs, - long aggreReleasedCtnrs, int availableMB, int availableCores, int pendingMB, - int pendingCores, int pendingCtnrs, int reservedMB, int reservedCores, - int reservedCtnrs) { + int allocatedCores, int allocatedVdisks, int allocCtnrs, + long aggreAllocCtnrs, long aggreReleasedCtnrs, int availableMB, + int availableCores, int availableVdisks, int pendingMB, int pendingCores, + int pendingVdisks, int pendingCtnrs, int reservedMB, int reservedCores, + int reservedVdisks, int reservedCtnrs) { MetricsRecordBuilder rb = getMetrics(source); assertGauge("AllocatedMB", allocatedMB, rb); assertGauge("AllocatedVCores", allocatedCores, rb); + assertGauge("AllocatedVDisks", allocatedVdisks, rb); assertGauge("AllocatedContainers", allocCtnrs, rb); assertCounter("AggregateContainersAllocated", aggreAllocCtnrs, rb); assertCounter("AggregateContainersReleased", aggreReleasedCtnrs, rb); assertGauge("AvailableMB", availableMB, rb); assertGauge("AvailableVCores", availableCores, rb); + assertGauge("AvailableVDisks", availableVdisks, rb); assertGauge("PendingMB", pendingMB, rb); assertGauge("PendingVCores", pendingCores, rb); + assertGauge("PendingVDisks", pendingVdisks, rb); assertGauge("PendingContainers", pendingCtnrs, rb); assertGauge("ReservedMB", reservedMB, rb); assertGauge("ReservedVCores", reservedCores, rb); + assertGauge("ReservedVDisks", reservedVdisks, rb); assertGauge("ReservedContainers", reservedCtnrs, rb); }