diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java index aea931a..23409fc 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java @@ -527,8 +527,8 @@ public static JobStatus fromYarn(ApplicationReport application, jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers()); jobStatus.setReservedMem( (int)resourceUsageReport.getReservedResources().getMemorySize()); - jobStatus.setUsedMem( - (int) resourceUsageReport.getUsedResources().getMemorySize()); + jobStatus.setUsedMem((int) resourceUsageReport + .getGuaranteedResourcesUsed().getMemorySize()); } return jobStatus; } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java index 629a246..acd9c35 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java @@ -94,7 +94,7 @@ public void testFromYarn() throws Exception { appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); appUsageRpt.setReservedResources(r); - appUsageRpt.setUsedResources(r); + appUsageRpt.setGuaranteedResourcesUsed(r); applicationReport.setApplicationResourceUsageReport(appUsageRpt); JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile"); Assert.assertEquals(appStartTime, jobStatus.getStartTime()); @@ -133,7 +133,7 @@ public void testFromYarnApplicationReport() { appUsageRpt.setNumReservedContainers(1); appUsageRpt.setNumUsedContainers(3); appUsageRpt.setReservedResources(r); - appUsageRpt.setUsedResources(r); + appUsageRpt.setGuaranteedResourcesUsed(r); when(mockReport.getApplicationResourceUsageReport()).thenReturn(appUsageRpt); JobStatus status = TypeConverter.fromYarn(mockReport, jobFile); Assert.assertNotNull("fromYarn returned null status", status); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java index d6754f0..0506b27 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java @@ -137,7 +137,7 @@ private ApplicationReport getApplicationReport( Records.newRecord(Resource.class)); Mockito.when(appResources.getReservedResources()).thenReturn( Records.newRecord(Resource.class)); - Mockito.when(appResources.getUsedResources()).thenReturn( + Mockito.when(appResources.getGuaranteedResourcesUsed()).thenReturn( Records.newRecord(Resource.class)); Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn( appResources); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index d2e33ff..4517db9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -36,21 +36,27 @@ @Private @Unstable public static ApplicationResourceUsageReport newInstance( - int numUsedContainers, int numReservedContainers, Resource usedResources, + int numUsedContainers, int numReservedContainers, + Resource guaranteedResourcesUsed, Resource reservedResources, Resource neededResources, - Map resourceSecondsMap, float queueUsagePerc, - float clusterUsagePerc, Map preemtedResourceSecondsMap) { + Map guaranteedResourceSecondsMap, float queueUsagePerc, + float clusterUsagePerc, Map preemtedResourceSecondsMap, + Resource opportunisticResourcesUsed, + Map opportunisticResourcesSecondsMap) { + ApplicationResourceUsageReport report = Records.newRecord(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); report.setNumReservedContainers(numReservedContainers); - report.setUsedResources(usedResources); + report.setGuaranteedResourcesUsed(guaranteedResourcesUsed); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); - report.setResourceSecondsMap(resourceSecondsMap); + report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap); report.setQueueUsagePercentage(queueUsagePerc); report.setClusterUsagePercentage(clusterUsagePerc); report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap); + report.setOpportunisticResourcesUsed(opportunisticResourcesUsed); + report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap); return report; } @@ -87,16 +93,40 @@ public static ApplicationResourceUsageReport newInstance( public abstract void setNumReservedContainers(int num_reserved_containers); /** - * Get the used Resource. -1 for invalid/inaccessible reports. - * @return the used Resource + * Get the guaranteed Resource used. + * -1 for invalid/inaccessible reports. + * @return the guaranteed Resource used */ @Public @Stable + @Deprecated public abstract Resource getUsedResources(); + /** + * Get the guaranteed Resource used. + * -1 for invalid/inaccessible reports. + * @return the guaranteed Resource used + */ + @Public + @Unstable + public abstract Resource getGuaranteedResourcesUsed(); + + @Private + @Unstable + public abstract void setGuaranteedResourcesUsed(Resource resources); + + /** + * Get the opportunistic Resource used. + * -1 for invalid/inaccessible reports. + * @return the opportunistic Resource used + */ + @Public + @Unstable + public abstract Resource getOpportunisticResourcesUsed(); + @Private @Unstable - public abstract void setUsedResources(Resource resources); + public abstract void setOpportunisticResourcesUsed(Resource resources); /** * Get the reserved Resource. -1 for invalid/inaccessible reports. @@ -123,42 +153,84 @@ public static ApplicationResourceUsageReport newInstance( public abstract void setNeededResources(Resource needed_resources); /** - * Set the aggregated amount of memory (in megabytes) the application has - * allocated times the number of seconds the application has been running. - * @param memory_seconds the aggregated amount of memory seconds + * Set the aggregated amount of guaranteed memory (in megabytes) the + * application has allocated times the number of seconds the application + * has been running. + * @param memorySeconds the aggregated amount of guaranteed memory seconds */ @Private @Unstable - public abstract void setMemorySeconds(long memory_seconds); + public abstract void setGuaranteedMemorySeconds(long memorySeconds); /** - * Get the aggregated amount of memory (in megabytes) the application has - * allocated times the number of seconds the application has been running. - * @return the aggregated amount of memory seconds + * Get the aggregated amount of guaranteed memory (in megabytes) the + * application has allocated times the number of seconds the application + * has been running. + * @return the aggregated amount of guaranteed memory seconds + */ + @Public + @Unstable + public abstract long getGuaranteedMemorySeconds(); + + /** + * Get the aggregated amount of guaranteed memory (in megabytes) the + * application has allocated times the number of seconds the application + * has been running. + * @return the aggregated amount of guaranteed memory seconds */ @Public @Unstable + @Deprecated public abstract long getMemorySeconds(); /** - * Set the aggregated number of vcores that the application has allocated - * times the number of seconds the application has been running. - * @param vcore_seconds the aggregated number of vcore seconds + * Set the aggregated number of guaranteed vcores that the application has + * allocated times the number of seconds the application has been running. + * @param vcoreSeconds the aggregated number of guaranteed vcore seconds */ @Private @Unstable - public abstract void setVcoreSeconds(long vcore_seconds); + public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds); /** - * Get the aggregated number of vcores that the application has allocated - * times the number of seconds the application has been running. - * @return the aggregated number of vcore seconds + * Get the aggregated number of guaranteed vcores that the application has + * allocated times the number of seconds the application has been running. + * @return the aggregated number of guaranteed vcore seconds + */ + @Public + @Unstable + public abstract long getGuaranteedVcoreSeconds(); + + /** + * Get the aggregated number of guaranteed vcores that the application has + * allocated times the number of seconds the application has been running. + * @return the aggregated number of guaranteed vcore seconds */ @Public @Unstable + @Deprecated public abstract long getVcoreSeconds(); /** + * Get the aggregated amount of opportunistic memory (in megabytes) the + * application has allocated times the number of seconds the application + * has been running. + * @return the aggregated amount of opportunistic memory seconds + */ + @Public + @Unstable + public abstract long getOpportunisticMemorySeconds(); + + /** + * Get the aggregated number of opportunistic vcores that the application + * has allocated times the number of seconds the application has been running. + * @return the aggregated number of opportunistic vcore seconds + */ + @Public + @Unstable + public abstract long getOpportunisticVcoreSeconds(); + + /** * Get the percentage of resources of the queue that the app is using. * @return the percentage of resources of the queue that the app is using. */ @@ -231,23 +303,35 @@ public static ApplicationResourceUsageReport newInstance( public abstract long getPreemptedVcoreSeconds(); /** - * Get the aggregated number of resources that the application has + * Get the aggregated number of guaranteed resources that the application has * allocated times the number of seconds the application has been running. - * @return map containing the resource name and aggregated resource-seconds + * @return map containing the resource name and aggregated guaranteed + * resource-seconds */ @Public @Unstable + @Deprecated public abstract Map getResourceSecondsMap(); /** - * Set the aggregated number of resources that the application has + * Get the aggregated number of guaranteed resources that the application has + * allocated times the number of seconds the application has been running. + * @return map containing the resource name and aggregated guaranteed + * resource-seconds + */ + @Public + @Unstable + public abstract Map getGuaranteedResourceSecondsMap(); + + /** + * Set the aggregated number of guaranteed resources that the application has * allocated times the number of seconds the application has been running. * @param resourceSecondsMap map containing the resource name and aggregated - * resource-seconds + * guaranteed resource-seconds */ @Private @Unstable - public abstract void setResourceSecondsMap( + public abstract void setGuaranteedResourceSecondsMap( Map resourceSecondsMap); @@ -272,4 +356,24 @@ public abstract void setResourceSecondsMap( public abstract void setPreemptedResourceSecondsMap( Map preemptedResourceSecondsMap); + /** + * Get the aggregated number of opportunistic resources that the application + * has allocated times the number of seconds the application has been running. + * @return map containing the resource name and aggregated opportunistic + * resource-seconds + */ + @Public + @Unstable + public abstract Map getOpportunisticResourceSecondsMap(); + + /** + * Set the aggregated number of opportunistic resources that the application + * has allocated times the number of seconds the application has been running. + * @param opportunisticResourceSecondsMap map containing the resource name + * and aggregated opportunistic resource-seconds + */ + @Private + @Unstable + public abstract void setOpportunisticResourceSecondsMap( + Map opportunisticResourceSecondsMap); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 83db918..db518fe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -251,6 +251,8 @@ message ApplicationResourceUsageReportProto { optional int64 preempted_vcore_seconds = 11; repeated StringLongMapProto application_resource_usage_map = 12; repeated StringLongMapProto application_preempted_resource_usage_map = 13; + optional ResourceProto used_opportunistic_resources = 14; + repeated StringLongMapProto application_opportunistic_resource_usage_map = 15; } message ApplicationReportProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index fb08fcd..4cb67af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -973,15 +973,20 @@ private ApplicationReport printApplicationReport(String applicationId) private void printResourceUsage(PrintWriter appReportStr, ApplicationResourceUsageReport usageReport) { - appReportStr.print("\tAggregate Resource Allocation : "); + appReportStr.print("\tAggregate Guaranteed Resource Allocation : "); if (usageReport != null) { - appReportStr.println( - getResourceSecondsString(usageReport.getResourceSecondsMap())); + appReportStr.println(getResourceSecondsString( + usageReport.getGuaranteedResourceSecondsMap())); + appReportStr.print("\tAggregate Opportunistic Resource Allocation : "); + appReportStr.println(getResourceSecondsString( + usageReport.getOpportunisticResourceSecondsMap())); appReportStr.print("\tAggregate Resource Preempted : "); appReportStr.println(getResourceSecondsString( usageReport.getPreemptedResourceSecondsMap())); } else { appReportStr.println("N/A"); + appReportStr.print("\tAggregate Opportunistic Resource Allocation : "); + appReportStr.println("N/A"); appReportStr.print("\tAggregate Resource Preempted : "); appReportStr.println("N/A"); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java index d6c33f4..ba6eb3f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java @@ -170,13 +170,11 @@ public ColumnInformation(String header, String format, boolean display, appReport.getApplicationResourceUsageReport() .getNumReservedContainers(); displayStringsMap.put(Columns.RCONT, String.valueOf(reservedContainers)); - usedVirtualCores = - appReport.getApplicationResourceUsageReport().getUsedResources() - .getVirtualCores(); + usedVirtualCores = appReport.getApplicationResourceUsageReport() + .getGuaranteedResourcesUsed().getVirtualCores(); displayStringsMap.put(Columns.VCORES, String.valueOf(usedVirtualCores)); - usedMemory = - appReport.getApplicationResourceUsageReport().getUsedResources() - .getMemorySize() / 1024; + usedMemory = appReport.getApplicationResourceUsageReport() + .getGuaranteedResourcesUsed().getMemorySize() / 1024; displayStringsMap.put(Columns.MEM, String.valueOf(usedMemory) + "G"); reservedVirtualCores = appReport.getApplicationResourceUsageReport().getReservedResources() @@ -195,11 +193,11 @@ public ColumnInformation(String header, String format, boolean display, progress = appReport.getProgress() * 100; displayStringsMap.put(Columns.PROGRESS, String.format("%.2f", progress)); // store in GBSeconds - memorySeconds = - appReport.getApplicationResourceUsageReport().getMemorySeconds() / 1024; + memorySeconds = appReport.getApplicationResourceUsageReport() + .getGuaranteedMemorySeconds() / 1024; displayStringsMap.put(Columns.MEMSECS, String.valueOf(memorySeconds)); - vcoreSeconds = - appReport.getApplicationResourceUsageReport().getVcoreSeconds(); + vcoreSeconds = appReport.getApplicationResourceUsageReport() + .getGuaranteedVcoreSeconds(); displayStringsMap.put(Columns.VCORESECS, String.valueOf(vcoreSeconds)); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 8d537ca..86b3068 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -121,18 +121,27 @@ public void testGetApplicationReport() throws Exception { for (int i = 0; i < 2; ++i) { ApplicationCLI cli = createAndGetAppCLI(); ApplicationId applicationId = ApplicationId.newInstance(1234, 5); - Map resourceSecondsMap = new HashMap<>(); + Map guaranteedResourceSecondsMap = new HashMap<>(); + Map opportunisticResourceSecondsMap = new HashMap<>(); Map preemptedResoureSecondsMap = new HashMap<>(); - resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L); - resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L); + guaranteedResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 123456L); + guaranteedResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 4567L); + opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 123456L); + opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 4567L); preemptedResoureSecondsMap .put(ResourceInformation.MEMORY_MB.getName(), 1111L); preemptedResoureSecondsMap .put(ResourceInformation.VCORES.getName(), 2222L); ApplicationResourceUsageReport usageReport = i == 0 ? null : ApplicationResourceUsageReport - .newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0, - preemptedResoureSecondsMap); + .newInstance(2, 0, null, null, null, + guaranteedResourceSecondsMap, 0, 0, + preemptedResoureSecondsMap, null, + opportunisticResourceSecondsMap); ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, @@ -168,7 +177,9 @@ public void testGetApplicationReport() throws Exception { pw.println("\tTracking-URL : N/A"); pw.println("\tRPC Port : 124"); pw.println("\tAM Host : host"); - pw.println("\tAggregate Resource Allocation : " + + pw.println("\tAggregate Guaranteed Resource Allocation : " + + (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); + pw.println("\tAggregate Opportunistic Resource Allocation : " + (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds")); pw.println("\tAggregate Resource Preempted : " + (i == 0 ? "N/A" : "1111 MB-seconds, 2222 vcore-seconds")); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java index 14ede5d..8ff1631 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java @@ -40,12 +40,14 @@ ApplicationResourceUsageReportProto.Builder builder = null; boolean viaProto = false; - Resource usedResources; - Resource reservedResources; - Resource neededResources; + private Resource guaranteedResourceUsed; + private Resource reservedResources; + private Resource neededResources; + private Resource opportunisticResourceUsed; - private Map resourceSecondsMap; + private Map guaranteedResourceSecondsMap; private Map preemptedResourceSecondsMap; + private Map opportunisticResourceSecondsMap; public ApplicationResourceUsageReportPBImpl() { builder = ApplicationResourceUsageReportProto.newBuilder(); @@ -55,8 +57,9 @@ public ApplicationResourceUsageReportPBImpl( ApplicationResourceUsageReportProto proto) { this.proto = proto; viaProto = true; - getResourceSecondsMap(); + getGuaranteedResourceSecondsMap(); getPreemptedResourceSecondsMap(); + getOpportunisticResourceSecondsMap(); } public synchronized ApplicationResourceUsageReportProto getProto() { @@ -87,8 +90,13 @@ public String toString() { } private void mergeLocalToBuilder() { - if (this.usedResources != null) { - builder.setUsedResources(convertToProtoFormat(this.usedResources)); + if (this.guaranteedResourceUsed != null) { + builder.setUsedResources( + convertToProtoFormat(this.guaranteedResourceUsed)); + } + if (this.opportunisticResourceUsed != null) { + builder.setUsedOpportunisticResources( + convertToProtoFormat(this.opportunisticResourceUsed)); } if (this.reservedResources != null) { builder.setReservedResources( @@ -99,19 +107,28 @@ private void mergeLocalToBuilder() { } builder.clearApplicationResourceUsageMap(); builder.clearApplicationPreemptedResourceUsageMap(); + builder.clearApplicationOpportunisticResourceUsageMap(); if (preemptedResourceSecondsMap != null && !preemptedResourceSecondsMap .isEmpty()) { builder.addAllApplicationPreemptedResourceUsageMap(ProtoUtils .convertMapToStringLongMapProtoList(preemptedResourceSecondsMap)); } - if (resourceSecondsMap != null && !resourceSecondsMap.isEmpty()) { + if (guaranteedResourceSecondsMap != null && + !guaranteedResourceSecondsMap.isEmpty()) { builder.addAllApplicationResourceUsageMap( - ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); + ProtoUtils.convertMapToStringLongMapProtoList( + guaranteedResourceSecondsMap)); + } + if (opportunisticResourceSecondsMap != null && + !opportunisticResourceSecondsMap.isEmpty()) { + builder.addAllApplicationOpportunisticResourceUsageMap( + ProtoUtils.convertMapToStringLongMapProtoList( + opportunisticResourceSecondsMap)); } - builder.setMemorySeconds(this.getMemorySeconds()); - builder.setVcoreSeconds(this.getVcoreSeconds()); + builder.setMemorySeconds(this.getGuaranteedMemorySeconds()); + builder.setVcoreSeconds(this.getGuaranteedVcoreSeconds()); builder.setPreemptedMemorySeconds(this.getPreemptedMemorySeconds()); builder.setPreemptedVcoreSeconds(this.getPreemptedVcoreSeconds()); } @@ -157,24 +174,53 @@ public synchronized void setNumReservedContainers( } @Override + @Deprecated public synchronized Resource getUsedResources() { + return getGuaranteedResourcesUsed(); + } + + @Override + public synchronized Resource getGuaranteedResourcesUsed() { ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; - if (this.usedResources != null) { - return this.usedResources; + if (this.guaranteedResourceUsed != null) { + return this.guaranteedResourceUsed; } if (!p.hasUsedResources()) { return null; } - this.usedResources = convertFromProtoFormat(p.getUsedResources()); - return this.usedResources; + this.guaranteedResourceUsed = convertFromProtoFormat(p.getUsedResources()); + return this.guaranteedResourceUsed; } @Override - public synchronized void setUsedResources(Resource resources) { + public synchronized void setGuaranteedResourcesUsed(Resource resources) { maybeInitBuilder(); if (resources == null) builder.clearUsedResources(); - this.usedResources = resources; + this.guaranteedResourceUsed = resources; + } + + @Override + public synchronized Resource getOpportunisticResourcesUsed() { + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + if (this.opportunisticResourceUsed != null) { + return this.opportunisticResourceUsed; + } + if (!p.hasUsedOpportunisticResources()) { + return null; + } + this.opportunisticResourceUsed = + convertFromProtoFormat(p.getUsedOpportunisticResources()); + return this.opportunisticResourceUsed; + } + + @Override + public synchronized void setOpportunisticResourcesUsed(Resource resources) { + maybeInitBuilder(); + if (resources == null) { + builder.clearUsedOpportunisticResources(); + } + this.opportunisticResourceUsed = resources; } @Override @@ -220,14 +266,14 @@ public synchronized void setNeededResources(Resource reserved_resources) { } @Override - public synchronized void setMemorySeconds(long memory_seconds) { - getResourceSecondsMap() - .put(ResourceInformation.MEMORY_MB.getName(), memory_seconds); + public synchronized void setGuaranteedMemorySeconds(long memorySeconds) { + getGuaranteedResourceSecondsMap() + .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); } @Override - public synchronized long getMemorySeconds() { - Map tmp = getResourceSecondsMap(); + public synchronized long getGuaranteedMemorySeconds() { + Map tmp = getGuaranteedResourceSecondsMap(); if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { return tmp.get(ResourceInformation.MEMORY_MB.getName()); } @@ -235,20 +281,50 @@ public synchronized long getMemorySeconds() { } @Override - public synchronized void setVcoreSeconds(long vcore_seconds) { - getResourceSecondsMap() - .put(ResourceInformation.VCORES.getName(), vcore_seconds); + @Deprecated + public synchronized long getMemorySeconds() { + return getGuaranteedMemorySeconds(); + } + + @Override + public synchronized void setGuaranteedVcoreSeconds(long vcoreSeconds) { + getGuaranteedResourceSecondsMap() + .put(ResourceInformation.VCORES.getName(), vcoreSeconds); } @Override + public synchronized long getGuaranteedVcoreSeconds() { + Map tmp = getGuaranteedResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.VCORES.getName())) { + return tmp.get(ResourceInformation.VCORES.getName()); + } + return 0; + } + + @Override + @Deprecated public synchronized long getVcoreSeconds() { - Map tmp = getResourceSecondsMap(); + return getGuaranteedVcoreSeconds(); + } + + @Override + public synchronized long getOpportunisticMemorySeconds() { + Map tmp = getOpportunisticResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { + return tmp.get(ResourceInformation.MEMORY_MB.getName()); + } + return 0; + } + + @Override + public synchronized long getOpportunisticVcoreSeconds() { + Map tmp = getOpportunisticResourceSecondsMap(); if (tmp.containsKey(ResourceInformation.VCORES.getName())) { return tmp.get(ResourceInformation.VCORES.getName()); } return 0; } - + @Override public synchronized void setPreemptedMemorySeconds( long preemptedMemorySeconds) { @@ -314,41 +390,47 @@ public synchronized void setClusterUsagePercentage(float clusterUsagePerc) { } @Override - public synchronized void setResourceSecondsMap( + public synchronized void setGuaranteedResourceSecondsMap( Map resourceSecondsMap) { - this.resourceSecondsMap = resourceSecondsMap; + this.guaranteedResourceSecondsMap = resourceSecondsMap; if (resourceSecondsMap == null) { return; } if (!resourceSecondsMap .containsKey(ResourceInformation.MEMORY_MB.getName())) { - this.setMemorySeconds(0L); + this.setGuaranteedMemorySeconds(0L); } if (!resourceSecondsMap.containsKey(ResourceInformation.VCORES.getName())) { - this.setVcoreSeconds(0L); + this.setGuaranteedVcoreSeconds(0L); } } @Override + @Deprecated public synchronized Map getResourceSecondsMap() { - if (this.resourceSecondsMap != null) { - return this.resourceSecondsMap; + return getGuaranteedResourceSecondsMap(); + } + + @Override + public synchronized Map getGuaranteedResourceSecondsMap() { + if (this.guaranteedResourceSecondsMap != null) { + return this.guaranteedResourceSecondsMap; } ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; - this.resourceSecondsMap = ProtoUtils + this.guaranteedResourceSecondsMap = ProtoUtils .convertStringLongMapProtoListToMap( p.getApplicationResourceUsageMapList()); - if (!this.resourceSecondsMap + if (!this.guaranteedResourceSecondsMap .containsKey(ResourceInformation.MEMORY_MB.getName())) { - this.setMemorySeconds(p.getMemorySeconds()); + this.setGuaranteedMemorySeconds(p.getMemorySeconds()); } - if (!this.resourceSecondsMap + if (!this.guaranteedResourceSecondsMap .containsKey(ResourceInformation.VCORES.getName())) { - this.setVcoreSeconds(p.getVcoreSeconds()); + this.setGuaranteedVcoreSeconds(p.getVcoreSeconds()); } - this.setMemorySeconds(p.getMemorySeconds()); - this.setVcoreSeconds(p.getVcoreSeconds()); - return this.resourceSecondsMap; + this.setGuaranteedMemorySeconds(p.getMemorySeconds()); + this.setGuaranteedVcoreSeconds(p.getVcoreSeconds()); + return this.guaranteedResourceSecondsMap; } @Override @@ -389,4 +471,45 @@ public synchronized void setPreemptedResourceSecondsMap( this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds()); return this.preemptedResourceSecondsMap; } + + @Override + public synchronized Map getOpportunisticResourceSecondsMap() { + if (this.opportunisticResourceSecondsMap != null) { + return this.opportunisticResourceSecondsMap; + } + ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder; + this.opportunisticResourceSecondsMap = ProtoUtils + .convertStringLongMapProtoListToMap( + p.getApplicationOpportunisticResourceUsageMapList()); + if (!opportunisticResourceSecondsMap.containsKey( + ResourceInformation.MEMORY_MB.getName())) { + this.opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 0L); + } + if (!opportunisticResourceSecondsMap.containsKey( + ResourceInformation.VCORES.getName())) { + this.opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 0L); + } + return this.opportunisticResourceSecondsMap; + } + + @Override + public synchronized void setOpportunisticResourceSecondsMap( + Map opportunisticResourceSecondsMap) { + this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap; + if (opportunisticResourceSecondsMap == null) { + return; + } + if (!opportunisticResourceSecondsMap + .containsKey(ResourceInformation.MEMORY_MB.getName())) { + this.opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 0L); + } + if (!opportunisticResourceSecondsMap + .containsKey(ResourceInformation.VCORES.getName())) { + this.opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 0L); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 1c08844..010a0de 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.util.UnitsConversionUtil; +import java.util.Map; + /** * Resources is a computation class which provides a set of apis to do * mathematical operations on Resource object. @@ -556,4 +558,22 @@ public static Resource normalizeDown(ResourceCalculator calculator, Resource resource, Resource factor) { return calculator.normalizeDown(resource, factor); } + + /** + * Merge resource usage entries from a map to another map. + * @param mergeFrom the map to merge from + * @param mergeTo the map to merge to + */ + public static void mergeResourceSecondsMap(Map mergeFrom, + Map mergeTo) { + for (Map.Entry entry : mergeFrom.entrySet()) { + Long value = mergeTo.get(entry.getKey()); + if (value != null) { + value += entry.getValue(); + } else { + value = entry.getValue(); + } + mergeTo.put(entry.getKey(), value); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 0b57717..85cc006 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.timeline.NameValuePair; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -330,29 +329,19 @@ private static ApplicationReportExt convertToApplicationReport( .toString(); } - if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) { - long vcoreSeconds = parseLong(entityInfo, - ApplicationMetricsConstants.APP_CPU_METRICS); - long memorySeconds = parseLong(entityInfo, - ApplicationMetricsConstants.APP_MEM_METRICS); - long preemptedMemorySeconds = parseLong(entityInfo, - ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS); - long preemptedVcoreSeconds = parseLong(entityInfo, - ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS); - Map resourceSecondsMap = new HashMap<>(); - Map preemptedResoureSecondsMap = new HashMap<>(); - resourceSecondsMap - .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); - resourceSecondsMap - .put(ResourceInformation.VCORES.getName(), vcoreSeconds); - preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), - preemptedMemorySeconds); - preemptedResoureSecondsMap - .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds); - + if (entityInfo.containsKey( + ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)) { + Map guaranteedResourceSecondsMap = + extractGuaranteedResourceSecondsMap(entityInfo); + Map preemptedResoureSecondsMap = + extractPreemptedResourceSecondsMap(entityInfo); + Map opportunisticResourceSecondsMap = + extractOpportunisticResourceSecondsMap(entityInfo); appResources = ApplicationResourceUsageReport - .newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0, - preemptedResoureSecondsMap); + .newInstance(0, 0, null, null, null, + guaranteedResourceSecondsMap, 0, 0, + preemptedResoureSecondsMap, null, + opportunisticResourceSecondsMap); } if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { @@ -456,6 +445,54 @@ private static ApplicationReportExt convertToApplicationReport( amNodeLabelExpression), appViewACLs); } + private static Map extractGuaranteedResourceSecondsMap( + Map entityInfo) { + Map guaranteedResourceSecondsMap = new HashMap<>(); + + long guaranteedVcoreSeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS); + long guaranteedMemorySeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS); + guaranteedResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), guaranteedMemorySeconds); + guaranteedResourceSecondsMap.put(ResourceInformation.VCORES.getName(), + guaranteedVcoreSeconds); + + return guaranteedResourceSecondsMap; + } + + private static Map extractOpportunisticResourceSecondsMap( + Map entityInfo) { + Map opportunisticResourceSecondsMap = new HashMap<>(); + + long opportunisticVcoreSeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS); + long opportunisticMemorySeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS); + opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), + opportunisticMemorySeconds); + opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), opportunisticVcoreSeconds); + + return opportunisticResourceSecondsMap; + } + + private static Map extractPreemptedResourceSecondsMap( + Map entityInfo) { + Map preemptedResoureSecondsMap = new HashMap<>(); + + long preemptedMemorySeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS); + long preemptedVcoreSeconds = parseLong(entityInfo, + ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS); + preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), + preemptedMemorySeconds); + preemptedResoureSecondsMap + .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds); + + return preemptedResoureSecondsMap; + } private static long parseLong(Map entityInfo, String infoKey) { long result = 0; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index 7ef6eca..6cb21ff 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -152,9 +152,13 @@ public void testApplicationReport() throws IOException, YarnException { ApplicationReport appReport = response.getApplicationReport(); Assert.assertNotNull(appReport); Assert.assertEquals(123, appReport.getApplicationResourceUsageReport() - .getMemorySeconds()); + .getGuaranteedMemorySeconds()); Assert.assertEquals(345, appReport.getApplicationResourceUsageReport() - .getVcoreSeconds()); + .getGuaranteedVcoreSeconds()); + Assert.assertEquals(123, appReport.getApplicationResourceUsageReport() + .getOpportunisticMemorySeconds()); + Assert.assertEquals(345, appReport.getApplicationResourceUsageReport() + .getOpportunisticVcoreSeconds()); Assert.assertEquals("application_0_0001", appReport.getApplicationId() .toString()); Assert.assertEquals("test app type", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index ecaaf1e..35996aa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -250,9 +250,13 @@ public ApplicationReport run() throws Exception { ApplicationResourceUsageReport applicationResourceUsageReport = app.getApplicationResourceUsageReport(); Assert.assertEquals(123, - applicationResourceUsageReport.getMemorySeconds()); - Assert - .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds()); + applicationResourceUsageReport.getGuaranteedMemorySeconds()); + Assert.assertEquals(345, + applicationResourceUsageReport.getGuaranteedVcoreSeconds()); + Assert.assertEquals(123, + applicationResourceUsageReport.getOpportunisticMemorySeconds()); + Assert.assertEquals(345, + applicationResourceUsageReport.getOpportunisticVcoreSeconds()); long expectedPreemptMemSecs = 456; long expectedPreemptVcoreSecs = 789; if (i == 3) { @@ -534,8 +538,12 @@ private static TimelineEntity createApplicationTimelineEntity( Priority.newInstance(0)); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); - entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123); - entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345); + entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, 123); + entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, 345); + entityInfo.put( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, 123); + entityInfo.put( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, 345); if (!missingPreemptMetrics) { entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456); entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index 4cec409..b4123ec 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -70,12 +70,18 @@ public static final String STATE_EVENT_INFO = "YARN_APPLICATION_STATE"; - public static final String APP_CPU_METRICS = + public static final String APP_GUARANTEED_CPU_METRICS = "YARN_APPLICATION_CPU"; - public static final String APP_MEM_METRICS = + public static final String APP_GUARANTEED_MEM_METRICS = "YARN_APPLICATION_MEMORY"; + public static final String APP_OPPORTUNISTIC_CPU_METRICS = + "YARN_APPLICATION_OPPORTUNISTIC_CPU"; + + public static final String APP_OPPORTUNISTIC_MEM_METRICS = + "YARN_APPLICATION_OPPORTUNISTIC_MEMORY"; + public static final String APP_RESOURCE_PREEMPTED_CPU = "YARN_APPLICATION_RESOURCE_PREEMPTED_CPU"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 6abc617..860ebae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -466,19 +466,24 @@ public static ApplicationSubmissionContext newApplicationSubmissionContext( } public static ApplicationResourceUsageReport newApplicationResourceUsageReport( - int numUsedContainers, int numReservedContainers, Resource usedResources, + int numUsedContainers, int numReservedContainers, + Resource guaranteedResourcesUsed, Resource reservedResources, Resource neededResources, - Map resourceSecondsMap, - Map preemptedResourceSecondsMap) { + Map guaranteedResourceSecondsMap, + Map preemptedResourceSecondsMap, + Resource opportunisticResourcesUsed, + Map opportunisticResourcesSecondsMap) { ApplicationResourceUsageReport report = recordFactory.newRecordInstance(ApplicationResourceUsageReport.class); report.setNumUsedContainers(numUsedContainers); report.setNumReservedContainers(numReservedContainers); - report.setUsedResources(usedResources); + report.setGuaranteedResourcesUsed(guaranteedResourcesUsed); report.setReservedResources(reservedResources); report.setNeededResources(neededResources); - report.setResourceSecondsMap(resourceSecondsMap); + report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap); report.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap); + report.setOpportunisticResourcesUsed(opportunisticResourcesUsed); + report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap); return report; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index ac2f8da..fb856d9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.Times; @@ -61,6 +62,8 @@ protected int priority; private long allocatedCpuVcores; private long allocatedMemoryMB; + private long allocatedOpportunisticCpuVcores; + private long allocatedOpportunisticMemoryMB; private long reservedCpuVcores; private long reservedMemoryMB; protected boolean unmanagedApplication; @@ -98,15 +101,25 @@ public AppInfo(ApplicationReport app) { if (app.getApplicationResourceUsageReport() != null) { runningContainers = app.getApplicationResourceUsageReport() .getNumUsedContainers(); - if (app.getApplicationResourceUsageReport().getUsedResources() != null) { - allocatedCpuVcores = app.getApplicationResourceUsageReport() - .getUsedResources().getVirtualCores(); - allocatedMemoryMB = app.getApplicationResourceUsageReport() - .getUsedResources().getMemorySize(); - reservedCpuVcores = app.getApplicationResourceUsageReport() - .getReservedResources().getVirtualCores(); - reservedMemoryMB = app.getApplicationResourceUsageReport() - .getReservedResources().getMemorySize(); + Resource guaranteedResourceUsed = + app.getApplicationResourceUsageReport().getGuaranteedResourcesUsed(); + if (guaranteedResourceUsed != null) { + allocatedCpuVcores = guaranteedResourceUsed.getVirtualCores(); + allocatedMemoryMB = guaranteedResourceUsed.getMemorySize(); + } + Resource opportunisticResourceUsed = app + .getApplicationResourceUsageReport().getOpportunisticResourcesUsed(); + if (opportunisticResourceUsed != null) { + allocatedOpportunisticCpuVcores = + opportunisticResourceUsed.getVirtualCores(); + allocatedOpportunisticMemoryMB = + opportunisticResourceUsed.getMemorySize(); + } + Resource resourceReserved = + app.getApplicationResourceUsageReport().getReservedResources(); + if (resourceReserved != null) { + reservedCpuVcores = resourceReserved.getVirtualCores(); + reservedMemoryMB = resourceReserved.getMemorySize(); } } progress = app.getProgress() * 100; // in percent @@ -166,6 +179,14 @@ public long getAllocatedMemoryMB() { return allocatedMemoryMB; } + public long getAllocatedOpportunisticCpuVcores() { + return allocatedOpportunisticCpuVcores; + } + + public long getAllocatedOpportunisticMemoryMB() { + return allocatedOpportunisticMemoryMB; + } + public long getReservedCpuVcores() { return reservedCpuVcores; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 1e05fec..cb7d426 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -187,19 +187,27 @@ public static SummaryBuilder createAppSummary(RMApp app) { .add("startTime", app.getStartTime()) .add("finishTime", app.getFinishTime()) .add("finalStatus", app.getFinalApplicationStatus()) - .add("memorySeconds", metrics.getMemorySeconds()) - .add("vcoreSeconds", metrics.getVcoreSeconds()) + .add("guaranteedMemorySeconds", metrics.getGuaranteedMemorySeconds()) + .add("guaranteedVcoreSeconds", metrics.getGuaranteedVcoreSeconds()) + .add("opportunisticMemorySeconds", + metrics.getOpportunisticMemorySeconds()) + .add("opportunisticVcoreSeconds", + metrics.getOpportunisticVcoreSeconds()) .add("preemptedMemorySeconds", metrics.getPreemptedMemorySeconds()) .add("preemptedVcoreSeconds", metrics.getPreemptedVcoreSeconds()) .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) .add("preemptedResources", metrics.getResourcePreempted()) .add("applicationType", app.getApplicationType()) - .add("resourceSeconds", StringHelper - .getResourceSecondsString(metrics.getResourceSecondsMap())) + .add("guaranteedResourceSeconds", StringHelper + .getResourceSecondsString( + metrics.getGuaranteedResourceSecondsMap())) .add("preemptedResourceSeconds", StringHelper .getResourceSecondsString( - metrics.getPreemptedResourceSecondsMap())); + metrics.getPreemptedResourceSecondsMap())) + .add("opportunisticResourceSeconds", StringHelper + .getResourceSecondsString( + metrics.getOpportunisticResourceSecondsMap())); return summary; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 4934243..05b37cd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -482,7 +482,8 @@ public static YarnApplicationAttemptState createApplicationAttemptState( DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = BuilderUtils.newApplicationResourceUsageReport(-1, -1, Resources.createResource(-1, -1), Resources.createResource(-1, -1), - Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>()); + Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>(), + Resources.createResource(-1, -1), new HashMap<>()); /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index 73bb301..efe5239 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -140,10 +140,16 @@ public void appFinished(RMApp app, RMAppState state, long finishedTime) { latestApplicationAttemptId); } RMAppMetrics appMetrics = app.getRMAppMetrics(); - entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS, - appMetrics.getVcoreSeconds()); - entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, - appMetrics.getMemorySeconds()); + entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, + appMetrics.getGuaranteedVcoreSeconds()); + entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, + appMetrics.getGuaranteedMemorySeconds()); + entity.addOtherInfo( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, + appMetrics.getOpportunisticVcoreSeconds()); + entity.addOtherInfo( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, + appMetrics.getOpportunisticMemorySeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, appMetrics.getPreemptedMemorySeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 7eaa6e7..84f3e9d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -190,11 +190,17 @@ public void appFinished(RMApp app, RMAppState state, long finishedTime) { Set entityMetrics = new HashSet(); entityMetrics.add(getTimelineMetric( - ApplicationMetricsConstants.APP_CPU_METRICS, timestamp, - appMetrics.getVcoreSeconds())); + ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, timestamp, + appMetrics.getGuaranteedVcoreSeconds())); entityMetrics.add(getTimelineMetric( - ApplicationMetricsConstants.APP_MEM_METRICS, timestamp, - appMetrics.getMemorySeconds())); + ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, timestamp, + appMetrics.getGuaranteedMemorySeconds())); + entityMetrics.add(getTimelineMetric( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, + timestamp, appMetrics.getOpportunisticVcoreSeconds())); + entityMetrics.add(getTimelineMetric( + ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, + timestamp, appMetrics.getOpportunisticMemorySeconds())); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, timestamp, appMetrics.getPreemptedMemorySeconds())); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index f0ab324..de66802 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -853,8 +853,9 @@ public void storeNewApplicationAttempt(RMAppAttempt appAttempt) { appAttempt.getAppAttemptId(), appAttempt.getMasterContainer(), credentials, appAttempt.getStartTime(), - resUsage.getResourceUsageSecondsMap(), - attempMetrics.getPreemptedResourceSecondsMap()); + resUsage.getGuaranteedResourceUsageSecondsMap(), + attempMetrics.getPreemptedResourceSecondsMap(), + resUsage.getOpportunisticResourceSecondsMap()); getRMStateStoreEventHandler().handle( new RMStateStoreAppAttemptEvent(attemptState)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java index 2de071a..b08d2eb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery.records; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -45,8 +46,9 @@ public static ApplicationAttemptStateData newInstance( Credentials attemptTokens, long startTime, RMAppAttemptState finalState, String finalTrackingUrl, String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus, - long finishTime, Map resourceSecondsMap, - Map preemptedResourceSecondsMap) { + long finishTime, Map guaranteedResourceSecondsMap, + Map preemptedResourceSecondsMap, + Map opportunisticResourcesSecondsMap) { ApplicationAttemptStateData attemptStateData = Records.newRecord(ApplicationAttemptStateData.class); attemptStateData.setAttemptId(attemptId); @@ -59,19 +61,22 @@ public static ApplicationAttemptStateData newInstance( attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus); attemptStateData.setAMContainerExitStatus(exitStatus); attemptStateData.setFinishTime(finishTime); - attemptStateData.setMemorySeconds(RMServerUtils - .getOrDefault(resourceSecondsMap, + attemptStateData.setGuaranteedMemorySeconds(RMServerUtils + .getOrDefault(guaranteedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L)); - attemptStateData.setVcoreSeconds(RMServerUtils - .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), - 0L)); + attemptStateData.setGuaranteedVcoreSeconds(RMServerUtils + .getOrDefault(guaranteedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L)); attemptStateData.setPreemptedMemorySeconds(RMServerUtils .getOrDefault(preemptedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L)); attemptStateData.setPreemptedVcoreSeconds(RMServerUtils .getOrDefault(preemptedResourceSecondsMap, ResourceInformation.VCORES.getName(), 0L)); - attemptStateData.setResourceSecondsMap(resourceSecondsMap); + attemptStateData.setGuaranteedResourceSecondsMap( + guaranteedResourceSecondsMap); + attemptStateData.setOpportunisticResourceSecondsMap( + opportunisticResourcesSecondsMap); attemptStateData .setPreemptedResourceSecondsMap(preemptedResourceSecondsMap); return attemptStateData; @@ -80,11 +85,13 @@ public static ApplicationAttemptStateData newInstance( public static ApplicationAttemptStateData newInstance( ApplicationAttemptId attemptId, Container masterContainer, Credentials attemptTokens, long startTime, - Map resourceSeondsMap, - Map preemptedResourceSecondsMap) { + Map guaranteedResourceSecondsMap, + Map preemptedResourceSecondsMap, + Map opportunisticResourcesSecondsMap) { return newInstance(attemptId, masterContainer, attemptTokens, startTime, null, "N/A", "", null, ContainerExitStatus.INVALID, 0, - resourceSeondsMap, preemptedResourceSecondsMap); + guaranteedResourceSecondsMap, preemptedResourceSecondsMap, + opportunisticResourcesSecondsMap); } @@ -180,30 +187,78 @@ public abstract void setFinalApplicationStatus( public abstract void setFinishTime(long finishTime); /** - * Get the memory seconds (in MB seconds) of the application. - * @return memory seconds (in MB seconds) of the application + * Get the guaranteed memory seconds (in MB seconds) of the + * application. + * @return guaranteed memory seconds (in MB seconds) of the + * application */ @Public @Unstable + @Deprecated public abstract long getMemorySeconds(); @Public @Unstable + @Deprecated public abstract void setMemorySeconds(long memorySeconds); /** - * Get the vcore seconds of the application. - * @return vcore seconds of the application + * Get the guaranteed vcore seconds of the application. + * @return guaranteed vcore seconds of the application */ @Public @Unstable + @Deprecated public abstract long getVcoreSeconds(); @Public @Unstable + @Deprecated public abstract void setVcoreSeconds(long vcoreSeconds); /** + * Get the guaranteed memory seconds (in MB seconds) of the + * application. + * @return guaranteed memory seconds (in MB seconds) of the + * application + */ + @Public + @Unstable + public abstract long getGuaranteedMemorySeconds(); + + @Private + public abstract void setGuaranteedMemorySeconds(long memorySeconds); + + /** + * Get the guaranteed vcore seconds of the application. + * @return guaranteed vcore seconds of the application + */ + @Public + @Unstable + public abstract long getGuaranteedVcoreSeconds(); + + @Private + public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds); + + /** + * Get the opportunistic memory seconds (in MB seconds) of the + * application. + * @return opportunistic memory seconds (in MB seconds) of the + * application + */ + @Public + @Unstable + public abstract long getOpportunisticMemorySeconds(); + + /** + * Get the opportunistic vcore seconds of the application. + * @return opportunistic vcore seconds of the application + */ + @Public + @Unstable + public abstract long getOpportunisticVcoreSeconds(); + + /** * Get the preempted memory seconds * (in MB seconds) of the application. * @return preempted memory seconds @@ -232,28 +287,54 @@ public abstract void setFinalApplicationStatus( public abstract void setPreemptedVcoreSeconds(long vcoreSeconds); /** - * Get the aggregated number of resources preempted that the application has + * Get the aggregated number of guaranteed resources that the application has * allocated times the number of seconds the application has been running. * - * @return map containing the resource name and aggregated preempted - * resource-seconds + * @return map containing the resource name and aggregated guaranteed + * resource-seconds */ @Public @Unstable + @Deprecated public abstract Map getResourceSecondsMap(); /** - * Set the aggregated number of resources that the application has + * Set the aggregated number of guaranteed resources that the application has * allocated times the number of seconds the application has been running. * * @param resourceSecondsMap map containing the resource name and aggregated - * resource-seconds + * guaranteed resource-seconds */ @Public @Unstable + @Deprecated public abstract void setResourceSecondsMap( Map resourceSecondsMap); + + /** + * Get the aggregated number of guaranteed resources that the application has + * allocated times the number of seconds the application has been running. + * + * @return map containing the resource name and aggregated guaranteed + * resource-seconds + */ + @Public + @Unstable + public abstract Map getGuaranteedResourceSecondsMap(); + + /** + * Set the aggregated number of guaranteed resources that the application has + * allocated times the number of seconds the application has been running. + * + * @param resourceSecondsMap map containing the resource name and aggregated + * guaranteed resource-seconds + */ + @Public + @Unstable + public abstract void setGuaranteedResourceSecondsMap( + Map resourceSecondsMap); + /** * Get the aggregated number of resources preempted that the application has * allocated times the number of seconds the application has been running. @@ -276,4 +357,27 @@ public abstract void setResourceSecondsMap( @Unstable public abstract void setPreemptedResourceSecondsMap( Map preemptedResourceSecondsMap); + + /** + * Get the aggregated number of opportunistic resources that the application + * has allocated times the number of seconds the application has been running. + * + * @return map containing the resource name and aggregated opportunistic + * resource-seconds + */ + @Public + @Unstable + public abstract Map getOpportunisticResourceSecondsMap(); + + /** + * Set the aggregated number of opportunistic resources that the application + * has allocated times the number of seconds the application has been running. + * + * @param resourceSecondsMap map containing the resource name and aggregated + * opportunistic resource-seconds + */ + @Public + @Unstable + public abstract void setOpportunisticResourceSecondsMap( + Map resourceSecondsMap); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java index ed71ea2..0243443 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; @@ -56,8 +57,9 @@ private Container masterContainer = null; private ByteBuffer appAttemptTokens = null; - private Map resourceSecondsMap; + private Map guaranteedResourceSecondsMap; private Map preemptedResourceSecondsMap; + private Map opportunisticResourceSecondsMap; public ApplicationAttemptStateDataPBImpl() { builder = ApplicationAttemptStateDataProto.newBuilder(); @@ -243,30 +245,72 @@ public void setStartTime(long startTime) { } @Override + @Deprecated public long getMemorySeconds() { - ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getMemorySeconds(); + return getGuaranteedMemorySeconds(); } @Override + @Deprecated public long getVcoreSeconds() { - ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; - return p.getVcoreSeconds(); + return getGuaranteedVcoreSeconds(); } @Override + @Deprecated public void setMemorySeconds(long memorySeconds) { - maybeInitBuilder(); - builder.setMemorySeconds(memorySeconds); + setGuaranteedMemorySeconds(memorySeconds); } @Override + @Deprecated public void setVcoreSeconds(long vcoreSeconds) { + setGuaranteedVcoreSeconds(vcoreSeconds); + } + + @Override + public long getGuaranteedMemorySeconds() { + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getMemorySeconds(); + } + + @Override + public void setGuaranteedMemorySeconds(long memorySeconds) { + maybeInitBuilder(); + builder.setMemorySeconds(memorySeconds); + } + + @Override + public long getGuaranteedVcoreSeconds() { + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getVcoreSeconds(); + } + + @Override + public void setGuaranteedVcoreSeconds(long vcoreSeconds) { maybeInitBuilder(); builder.setVcoreSeconds(vcoreSeconds); } @Override + public long getOpportunisticMemorySeconds() { + Map tmp = getOpportunisticResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) { + return tmp.get(ResourceInformation.MEMORY_MB.getName()); + } + return 0; + } + + @Override + public long getOpportunisticVcoreSeconds() { + Map tmp = getOpportunisticResourceSecondsMap(); + if (tmp.containsKey(ResourceInformation.VCORES.getName())) { + return tmp.get(ResourceInformation.VCORES.getName()); + } + return 0; + } + + @Override public long getPreemptedMemorySeconds() { ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; return p.getPreemptedMemorySeconds(); @@ -410,21 +454,35 @@ private static Credentials convertCredentialsFromByteBuffer( } @Override + @Deprecated public Map getResourceSecondsMap() { - if (this.resourceSecondsMap != null) { - return this.resourceSecondsMap; + return getGuaranteedResourceSecondsMap(); + } + + @Override + @Deprecated + public void setResourceSecondsMap(Map resourceSecondsMap) { + setGuaranteedResourceSecondsMap(resourceSecondsMap); + } + + @Override + public Map getGuaranteedResourceSecondsMap() { + if (this.guaranteedResourceSecondsMap != null) { + return this.guaranteedResourceSecondsMap; } ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; - this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap( - p.getApplicationResourceUsageMapList()); - return this.resourceSecondsMap; + this.guaranteedResourceSecondsMap = + ProtoUtils.convertStringLongMapProtoListToMap( + p.getApplicationResourceUsageMapList()); + return this.guaranteedResourceSecondsMap; } @Override - public void setResourceSecondsMap(Map resourceSecondsMap) { + public void setGuaranteedResourceSecondsMap( + Map resourceSecondsMap) { maybeInitBuilder(); builder.clearApplicationResourceUsageMap(); - this.resourceSecondsMap = resourceSecondsMap; + this.guaranteedResourceSecondsMap = resourceSecondsMap; if (resourceSecondsMap != null) { builder.addAllApplicationResourceUsageMap( ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); @@ -454,4 +512,28 @@ public void setPreemptedResourceSecondsMap( .convertMapToStringLongMapProtoList(preemptedResourceSecondsMap)); } } + + @Override + public Map getOpportunisticResourceSecondsMap() { + if (this.opportunisticResourceSecondsMap != null) { + return this.opportunisticResourceSecondsMap; + } + ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder; + this.opportunisticResourceSecondsMap = + ProtoUtils.convertStringLongMapProtoListToMap( + p.getApplicationOpportunisticResourceUsageMapList()); + return this.opportunisticResourceSecondsMap; + } + + @Override + public void setOpportunisticResourceSecondsMap( + Map resourceSecondsMap) { + maybeInitBuilder(); + builder.clearApplicationOpportunisticResourceUsageMap(); + this.opportunisticResourceSecondsMap = resourceSecondsMap; + if (resourceSecondsMap != null) { + builder.addAllApplicationOpportunisticResourceUsageMap( + ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap)); + } + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 38f666b..dc69af2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -449,6 +449,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, } else { this.startTime = startTime; } + LOG.info(applicationId.toString() + " started at " + this.startTime); this.applicationType = StringInterner.weakIntern(applicationType); this.applicationTags = applicationTags; this.amReqs = amReqs; @@ -754,10 +755,12 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, } RMAppMetrics rmAppMetrics = getRMAppMetrics(); - appUsageReport - .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap()); + appUsageReport.setGuaranteedResourceSecondsMap( + rmAppMetrics.getGuaranteedResourceSecondsMap()); appUsageReport.setPreemptedResourceSecondsMap( rmAppMetrics.getPreemptedResourceSecondsMap()); + appUsageReport.setOpportunisticResourceSecondsMap( + rmAppMetrics.getOpportunisticResourceSecondsMap()); } if (currentApplicationAttemptId == null) { @@ -1627,8 +1630,9 @@ public RMAppMetrics getRMAppMetrics() { Resource resourcePreempted = Resource.newInstance(0, 0); int numAMContainerPreempted = 0; int numNonAMContainerPreempted = 0; - Map resourceSecondsMap = new HashMap<>(); - Map preemptedSecondsMap = new HashMap<>(); + Map guaranteedResourceSecondsMap = new HashMap<>(2); + Map preemptedSecondsMap = new HashMap<>(2); + Map opportunsiticResourceSecondsMap = new HashMap<>(2); this.readLock.lock(); try { for (RMAppAttempt attempt : attempts.values()) { @@ -1644,20 +1648,15 @@ public RMAppMetrics getRMAppMetrics() { // for both running and finished containers. AggregateAppResourceUsage resUsage = attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage(); - for (Map.Entry entry : resUsage - .getResourceUsageSecondsMap().entrySet()) { - long value = RMServerUtils - .getOrDefault(resourceSecondsMap, entry.getKey(), 0L); - value += entry.getValue(); - resourceSecondsMap.put(entry.getKey(), value); - } - for (Map.Entry entry : attemptMetrics - .getPreemptedResourceSecondsMap().entrySet()) { - long value = RMServerUtils - .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L); - value += entry.getValue(); - preemptedSecondsMap.put(entry.getKey(), value); - } + Resources.mergeResourceSecondsMap( + resUsage.getGuaranteedResourceUsageSecondsMap(), + guaranteedResourceSecondsMap); + Resources.mergeResourceSecondsMap( + resUsage.getOpportunisticResourceSecondsMap(), + opportunsiticResourceSecondsMap); + Resources.mergeResourceSecondsMap( + attemptMetrics.getPreemptedResourceSecondsMap(), + preemptedSecondsMap); } } } finally { @@ -1665,7 +1664,8 @@ public RMAppMetrics getRMAppMetrics() { } return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted, - numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap); + numAMContainerPreempted, guaranteedResourceSecondsMap, + preemptedSecondsMap, opportunsiticResourceSecondsMap); } @Private diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java index 2bb7fd1..76272e9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java @@ -28,18 +28,21 @@ final Resource resourcePreempted; final int numNonAMContainersPreempted; final int numAMContainersPreempted; - private final Map resourceSecondsMap; + private final Map guaranteedResourceSecondsMap; private final Map preemptedResourceSecondsMap; + private final Map opportunisticResourceSecondsMap; public RMAppMetrics(Resource resourcePreempted, int numNonAMContainersPreempted, int numAMContainersPreempted, - Map resourceSecondsMap, - Map preemptedResourceSecondsMap) { + Map guaranteedResourceSecondsMap, + Map preemptedResourceSecondsMap, + Map opportunisticResourceSecondsMap) { this.resourcePreempted = resourcePreempted; this.numNonAMContainersPreempted = numNonAMContainersPreempted; this.numAMContainersPreempted = numAMContainersPreempted; - this.resourceSecondsMap = resourceSecondsMap; + this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap; this.preemptedResourceSecondsMap = preemptedResourceSecondsMap; + this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap; } public Resource getResourcePreempted() { @@ -54,17 +57,25 @@ public int getNumAMContainersPreempted() { return numAMContainersPreempted; } - public long getMemorySeconds() { - return RMServerUtils.getOrDefault(resourceSecondsMap, + public long getGuaranteedMemorySeconds() { + return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L); } - public long getVcoreSeconds() { - return RMServerUtils - .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), - 0L); + public long getGuaranteedVcoreSeconds() { + return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); + } + + public long getOpportunisticMemorySeconds() { + return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); } + public long getOpportunisticVcoreSeconds() { + return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); + } public long getPreemptedMemorySeconds() { return RMServerUtils.getOrDefault(preemptedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L); @@ -75,12 +86,15 @@ public long getPreemptedVcoreSeconds() { ResourceInformation.VCORES.getName(), 0L); } - public Map getResourceSecondsMap() { - return resourceSecondsMap; + public Map getGuaranteedResourceSecondsMap() { + return guaranteedResourceSecondsMap; } public Map getPreemptedResourceSecondsMap() { return preemptedResourceSecondsMap; } + public Map getOpportunisticResourceSecondsMap() { + return opportunisticResourceSecondsMap; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java index b858712..a75cc57 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java @@ -27,30 +27,55 @@ @Private public class AggregateAppResourceUsage { - private Map resourceSecondsMap = new HashMap<>(); + private Map guaranteedResourceSecondsMap = new HashMap<>(); + private Map opportunisticResourceSecondsMap = new HashMap<>(); - public AggregateAppResourceUsage(Map resourceSecondsMap) { - this.resourceSecondsMap.putAll(resourceSecondsMap); + public AggregateAppResourceUsage( + Map guaranteedResourceSecondsMap, + Map opportunisticResourceSecondsMap) { + this.guaranteedResourceSecondsMap.putAll( + guaranteedResourceSecondsMap); + this.opportunisticResourceSecondsMap.putAll( + opportunisticResourceSecondsMap); } /** - * @return the memorySeconds + * @return the guaranteed memory seconds */ - public long getMemorySeconds() { - return RMServerUtils.getOrDefault(resourceSecondsMap, + public long getGuaranteedMemorySeconds() { + return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L); } /** - * @return the vcoreSeconds + * @return the guaranteed vcore seconds */ - public long getVcoreSeconds() { - return RMServerUtils - .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), - 0L); + public long getGuaranteedVcoreSeconds() { + return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); } - public Map getResourceUsageSecondsMap() { - return resourceSecondsMap; + /** + * @return the opportunistic memory seconds + */ + public long getOpportunisticMemorySeconds() { + return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap, + ResourceInformation.MEMORY_MB.getName(), 0L); + } + + /** + * @return the opportunistic vcore seconds + */ + public long getOpportunisticVcoreSeconds() { + return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap, + ResourceInformation.VCORES.getName(), 0L); + } + + public Map getGuaranteedResourceUsageSecondsMap() { + return guaranteedResourceSecondsMap; + } + + public Map getOpportunisticResourceSecondsMap() { + return opportunisticResourceSecondsMap; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 2d5c6a3..d716e7f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -933,15 +933,18 @@ public ApplicationResourceUsageReport getApplicationResourceUsageReport() { this.readLock.lock(); try { ApplicationResourceUsageReport report = - scheduler.getAppResourceUsageReport(this.getAppAttemptId()); + scheduler.getAppActiveResourceUsageReport(this.getAppAttemptId()); if (report == null) { report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; } AggregateAppResourceUsage resUsage = this.attemptMetrics.getAggregateAppResourceUsage(); - report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap()); + report.setGuaranteedResourceSecondsMap( + resUsage.getGuaranteedResourceUsageSecondsMap()); report.setPreemptedResourceSecondsMap( this.attemptMetrics.getPreemptedResourceSecondsMap()); + report.setOpportunisticResourceSecondsMap( + resUsage.getOpportunisticResourceSecondsMap()); return report; } finally { this.readLock.unlock(); @@ -978,10 +981,12 @@ public void recover(RMState state) { this.finalStatus = attemptState.getFinalApplicationStatus(); this.startTime = attemptState.getStartTime(); this.finishTime = attemptState.getFinishTime(); - this.attemptMetrics - .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap()); + this.attemptMetrics.updateAggregateAppGuaranteedResourceUsage( + attemptState.getGuaranteedResourceSecondsMap()); this.attemptMetrics.updateAggregatePreemptedAppResourceUsage( attemptState.getPreemptedResourceSecondsMap()); + this.attemptMetrics.updateAggregateAppOpportunisticResourceUsage( + attemptState.getOpportunisticResourceSecondsMap()); } public void transferStateFromAttempt(RMAppAttempt attempt) { @@ -1359,9 +1364,11 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, ApplicationAttemptStateData attemptState = ApplicationAttemptStateData .newInstance(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, - stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus, - getFinishTime(), resUsage.getResourceUsageSecondsMap(), - this.attemptMetrics.getPreemptedResourceSecondsMap()); + stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, + exitStatus, getFinishTime(), + resUsage.getGuaranteedResourceUsageSecondsMap(), + this.attemptMetrics.getPreemptedResourceSecondsMap(), + resUsage.getOpportunisticResourceSecondsMap()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index 015cff7..192722b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -53,8 +53,11 @@ private ReadLock readLock; private WriteLock writeLock; - private Map resourceUsageMap = new HashMap<>(); + private Map guaranteedResourceUsageMap + = new HashMap<>(); private Map preemptedResourceMap = new HashMap<>(); + private Map opportunisticResourceUsageMap + = new HashMap<>(); private RMContext rmContext; private int[][] localityStatistics = @@ -129,8 +132,10 @@ public boolean getIsPreempted() { } public AggregateAppResourceUsage getAggregateAppResourceUsage() { - Map resourcesUsed = - convertAtomicLongMaptoLongMap(resourceUsageMap); + Map guaranteedResourcesUsed = + convertAtomicLongMaptoLongMap(guaranteedResourceUsageMap); + Map opportunisticResourcesUsed = + convertAtomicLongMaptoLongMap(opportunisticResourceUsageMap); // Only add in the running containers if this is the active attempt. RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId()); @@ -139,27 +144,25 @@ public AggregateAppResourceUsage getAggregateAppResourceUsage() { if (currentAttempt != null && currentAttempt.getAppAttemptId().equals(attemptId)) { ApplicationResourceUsageReport appResUsageReport = - rmContext.getScheduler().getAppResourceUsageReport(attemptId); + rmContext.getScheduler().getAppActiveResourceUsageReport(attemptId); if (appResUsageReport != null) { - Map tmp = appResUsageReport.getResourceSecondsMap(); - for (Map.Entry entry : tmp.entrySet()) { - Long value = resourcesUsed.get(entry.getKey()); - if (value != null) { - value += entry.getValue(); - } else { - value = entry.getValue(); - } - resourcesUsed.put(entry.getKey(), value); - } + Resources.mergeResourceSecondsMap( + appResUsageReport.getGuaranteedResourceSecondsMap(), + guaranteedResourcesUsed); + Resources.mergeResourceSecondsMap( + appResUsageReport.getOpportunisticResourceSecondsMap(), + opportunisticResourcesUsed); } } } - return new AggregateAppResourceUsage(resourcesUsed); + + return new AggregateAppResourceUsage(guaranteedResourcesUsed, + opportunisticResourcesUsed); } public void updateAggregateAppResourceUsage(Resource allocated, long deltaUsedMillis) { - updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap); + updateUsageMap(allocated, deltaUsedMillis, guaranteedResourceUsageMap); } public void updateAggregatePreemptedAppResourceUsage(Resource allocated, @@ -167,9 +170,14 @@ public void updateAggregatePreemptedAppResourceUsage(Resource allocated, updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap); } - public void updateAggregateAppResourceUsage( + public void updateAggregateAppGuaranteedResourceUsage( Map resourceSecondsMap) { - updateUsageMap(resourceSecondsMap, resourceUsageMap); + updateUsageMap(resourceSecondsMap, guaranteedResourceUsageMap); + } + + public void updateAggregateAppOpportunisticResourceUsage( + Map resourceSecondsMap) { + updateUsageMap(resourceSecondsMap, opportunisticResourceUsageMap); } public void updateAggregatePreemptedAppResourceUsage( @@ -199,10 +207,9 @@ private void updateUsageMap(Map sourceMap, if (!targetMap.containsKey(entry.getKey())) { resourceUsed = new AtomicLong(0); targetMap.put(entry.getKey(), resourceUsed); - } resourceUsed = targetMap.get(entry.getKey()); - resourceUsed.set(entry.getValue()); + resourceUsed.addAndGet(entry.getValue()); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java new file mode 100644 index 0000000..c91dae8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +import java.util.Map; + +/** + * The resource usage information of a Container. + */ +@Private +public class ContainerResourceUsageReport { + private final Map guaranteedResourceSecondsMap; + private final Map opportunisticResourceSecondsMap; + + public ContainerResourceUsageReport( + Map guaranteedResourceSecondsMap, + Map opportunisticResourceSecondsMap) { + this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap; + this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap; + } + + public Map getGuaranteedResourceUsageSecondsMap() { + return guaranteedResourceSecondsMap; + } + + public Map getOpportunisticResourceSecondsMap() { + return opportunisticResourceSecondsMap; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java index f3cbf63..4c43c5a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java @@ -18,8 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; -import java.util.List; - import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -29,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; @@ -85,7 +82,9 @@ ContainerState getContainerState(); ContainerReport createContainerReport(); - + + ContainerResourceUsageReport getResourceUsageReport(); + boolean isAMContainer(); ContainerRequest getContainerRequest(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index e26689e..8f51d1e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -20,11 +20,13 @@ import java.util.Collections; import java.util.EnumSet; -import java.util.List; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import org.apache.commons.lang.time.DateUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -39,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; @@ -176,7 +178,7 @@ private NodeId reservedNode; private SchedulerRequestKey reservedSchedulerKey; private long creationTime; - private long finishTime; + private long finishTime = -1L; private ContainerStatus finishedStatus; private boolean isAMContainer; private ContainerRequest containerRequestForRecovery; @@ -189,6 +191,10 @@ private boolean isExternallyAllocated; private SchedulerRequestKey allocatedSchedulerKey; + // This is updated whenever the container instance is updated in + // the case of container increase/decrease/promotion/demotion + private long lastContainerUpdateTime; + public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey, ApplicationAttemptId appAttemptId, NodeId nodeId, String user, RMContext rmContext) { @@ -230,6 +236,7 @@ public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey, this.appAttemptId = appAttemptId; this.user = user; this.creationTime = creationTime; + this.lastContainerUpdateTime = creationTime; this.rmContext = rmContext; this.eventHandler = rmContext.getDispatcher().getEventHandler(); this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer(); @@ -277,9 +284,87 @@ public Container getContainer() { } public void setContainer(Container container) { + // containers are updated by resetting the underlying Container instance, + // always update RMContainer resource usage before the update. + updateAppAttemptMetrics(true); this.container = container; } + /** + * Collect the resource usage information of the current underlying Container + * instance and update its RMAppAttemptMetrics. This is called whenever the + * RMContainer is updated (by updating its underlying container) or finishes. + * @param containerUpdated if it is called upon container update event + */ + private void updateAppAttemptMetrics(boolean containerUpdated) { + RMAppAttempt rmAttempt = rmContext.getRMApps() + .get(getApplicationAttemptId().getApplicationId()) + .getCurrentAppAttempt(); + + if (rmAttempt != null) { + // collect resource usage information of the current Container instance + ContainerResourceUsageReport resourceUsage = + getCurrentContainerResourceUsage(containerUpdated); + + // If this is a preempted container, update preemption metrics + if (finishedStatus != null && + ContainerExitStatus.PREEMPTED == finishedStatus.getExitStatus()) { + rmAttempt.getRMAppAttemptMetrics() + .updatePreemptionInfo(container.getResource(), this); + rmAttempt.getRMAppAttemptMetrics() + .updateAggregateAppOpportunisticResourceUsage( + resourceUsage.getGuaranteedResourceUsageSecondsMap()); + } + rmAttempt.getRMAppAttemptMetrics() + .updateAggregateAppOpportunisticResourceUsage( + resourceUsage.getOpportunisticResourceSecondsMap()); + rmAttempt.getRMAppAttemptMetrics() + .updateAggregateAppGuaranteedResourceUsage( + resourceUsage.getGuaranteedResourceUsageSecondsMap()); + } + } + + /** + * Get resource usage of the current underlying Container instance. + * @param containerUpdated if this is called upon container update + * @return resource usage of the current container + */ + private ContainerResourceUsageReport getCurrentContainerResourceUsage( + boolean containerUpdated) { + ContainerResourceUsageReport report; + + // A container generates usage until it finishes which is indicated + // by a positive finish timestamp that is set once it finishes + final long currentTimeMillis = + finishTime < 0 ? System.currentTimeMillis() : finishTime; + final long usedSeconds = (currentTimeMillis - + lastContainerUpdateTime) / DateUtils.MILLIS_PER_SECOND; + Resource resource = container.getResource(); + + if (container.getExecutionType() == ExecutionType.GUARANTEED) { + Map guaranteedResourceSeconds = new HashMap<>(2); + for (ResourceInformation entry : resource.getResources()) { + guaranteedResourceSeconds.put( + entry.getName(), entry.getValue() * usedSeconds); + } + report = new ContainerResourceUsageReport(guaranteedResourceSeconds, + Collections.emptyMap()); + } else { + Map opportunisticResourceSeconds = new HashMap<>(2); + for (ResourceInformation entry : resource.getResources()) { + opportunisticResourceSeconds.put( + entry.getName(), entry.getValue() * usedSeconds); + } + report = new ContainerResourceUsageReport(Collections.emptyMap(), + opportunisticResourceSeconds); + } + + if (containerUpdated) { + lastContainerUpdateTime = currentTimeMillis; + } + return report; + } + @Override public RMContainerState getState() { this.readLock.lock(); @@ -683,7 +768,7 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { // Inform AppAttempt // container.getContainer() can return null when a RMContainer is a // reserved container - updateAttemptMetrics(container); + container.updateAppAttemptMetrics(false); container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( container.appAttemptId, finishedEvent.getRemoteContainerStatus(), @@ -705,27 +790,6 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { } } - - private static void updateAttemptMetrics(RMContainerImpl container) { - Resource resource = container.getContainer().getResource(); - RMAppAttempt rmAttempt = container.rmContext.getRMApps() - .get(container.getApplicationAttemptId().getApplicationId()) - .getCurrentAppAttempt(); - - if (rmAttempt != null) { - long usedMillis = container.finishTime - container.creationTime; - rmAttempt.getRMAppAttemptMetrics() - .updateAggregateAppResourceUsage(resource, usedMillis); - // If this is a preempted container, update preemption metrics - if (ContainerExitStatus.PREEMPTED == container.finishedStatus - .getExitStatus()) { - rmAttempt.getRMAppAttemptMetrics() - .updatePreemptionInfo(resource, container); - rmAttempt.getRMAppAttemptMetrics() - .updateAggregatePreemptedAppResourceUsage(resource, usedMillis); - } - } - } } private static final class KillTransition extends FinishedTransition { @@ -764,6 +828,11 @@ public ContainerReport createContainerReport() { } @Override + public ContainerResourceUsageReport getResourceUsageReport() { + return getCurrentContainerResourceUsage(false); + } + + @Override public String getNodeHttpAddress() { try { readLock.lock(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index a755a4c..b78e93e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -412,7 +412,7 @@ public SchedulerAppReport getSchedulerAppInfo( } @Override - public ApplicationResourceUsageReport getAppResourceUsageReport( + public ApplicationResourceUsageReport getAppActiveResourceUsageReport( ApplicationAttemptId appAttemptId) { SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId); if (attempt == null) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index dfb0e67..bfb160a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -34,7 +34,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang.time.FastDateFormat; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -60,12 +59,12 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.ContainerType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; @@ -107,7 +106,10 @@ private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000; protected long lastMemoryAggregateAllocationUpdateTime = 0; - private Map lastResourceSecondsMap = new HashMap<>(); + private Map lastGuaranteedResourceSecondsMap = + new HashMap<>(); + private Map lastOpportunisticResourceSecondsMap = + new HashMap<>(); protected final AppSchedulingInfo appSchedulingInfo; protected ApplicationAttemptId attemptId; protected Map liveContainers = @@ -1050,34 +1052,47 @@ private AggregateAppResourceUsage getRunningAggregateAppResourceUsage() { // recently. if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime) > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) { - Map resourceSecondsMap = new HashMap<>(); + Map aggregatedGuaranteedResourceSecondsMap = + new HashMap<>(); + Map aggregatedOpportunisticResourceSecondsMap = + new HashMap<>(); for (RMContainer rmContainer : this.liveContainers.values()) { - long usedMillis = currentTimeMillis - rmContainer.getCreationTime(); - Resource resource = rmContainer.getContainer().getResource(); - for (ResourceInformation entry : resource.getResources()) { - long value = RMServerUtils - .getOrDefault(resourceSecondsMap, entry.getName(), 0L); - value += entry.getValue() * usedMillis - / DateUtils.MILLIS_PER_SECOND; - resourceSecondsMap.put(entry.getName(), value); - } + ContainerResourceUsageReport containerResourceUsageReport = + rmContainer.getResourceUsageReport(); + Resources.mergeResourceSecondsMap( + containerResourceUsageReport.getGuaranteedResourceUsageSecondsMap(), + aggregatedGuaranteedResourceSecondsMap); + Resources.mergeResourceSecondsMap( + containerResourceUsageReport.getOpportunisticResourceSecondsMap(), + aggregatedOpportunisticResourceSecondsMap); } - + lastGuaranteedResourceSecondsMap = + aggregatedGuaranteedResourceSecondsMap; + lastOpportunisticResourceSecondsMap = + aggregatedOpportunisticResourceSecondsMap; lastMemoryAggregateAllocationUpdateTime = currentTimeMillis; - lastResourceSecondsMap = resourceSecondsMap; } - return new AggregateAppResourceUsage(lastResourceSecondsMap); + return new AggregateAppResourceUsage(lastGuaranteedResourceSecondsMap, + lastOpportunisticResourceSecondsMap); } + /** + * Get the resources that are actively being used by the app attempt. + * @return active resource usage + */ public ApplicationResourceUsageReport getResourceUsageReport() { try { writeLock.lock(); AggregateAppResourceUsage runningResourceUsage = getRunningAggregateAppResourceUsage(); - Resource usedResourceClone = Resources.clone( + Resource guaranteedResourceUsedClone = Resources.clone( attemptResourceUsage.getAllUsed()); + Resource opportunisticResourceUsedClone = Resources.clone( + attemptOpportunisticResourceUsage.getAllUsed()); Resource reservedResourceClone = Resources.clone( attemptResourceUsage.getReserved()); + Resource neededResource = Resources.add(opportunisticResourceUsedClone, + Resources.add(guaranteedResourceUsedClone, reservedResourceClone)); Resource cluster = rmContext.getScheduler().getClusterResource(); ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator(); @@ -1092,18 +1107,20 @@ public ApplicationResourceUsageReport getResourceUsageReport() { float queueCapacityPerc = queue.getQueueInfo(false, false) .getCapacity(); if (queueCapacityPerc != 0) { - queueUsagePerc = calc.divide(cluster, usedResourceClone, + queueUsagePerc = calc.divide(cluster, guaranteedResourceUsedClone, Resources.multiply(cluster, queueCapacityPerc)) * 100; } clusterUsagePerc = - calc.divide(cluster, usedResourceClone, cluster) * 100; + calc.divide(cluster, guaranteedResourceUsedClone, cluster) * 100; } return ApplicationResourceUsageReport .newInstance(liveContainers.size(), reservedContainers.size(), - usedResourceClone, reservedResourceClone, - Resources.add(usedResourceClone, reservedResourceClone), - runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc, - clusterUsagePerc, preemptedResourceSecondsMaps); + guaranteedResourceUsedClone, reservedResourceClone, + neededResource, + runningResourceUsage.getGuaranteedResourceUsageSecondsMap(), + queueUsagePerc, clusterUsagePerc, preemptedResourceSecondsMaps, + opportunisticResourceUsedClone, + runningResourceUsage.getOpportunisticResourceSecondsMap()); } finally { writeLock.unlock(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 93ca7c2..a8ec8b3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -171,7 +171,7 @@ Allocation allocate(ApplicationAttemptId appAttemptId, */ @LimitedPrivate("yarn") @Evolving - ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationResourceUsageReport getAppActiveResourceUsageReport( ApplicationAttemptId appAttemptId); /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 21d6794..934c3a1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -1015,7 +1015,7 @@ public ApplicationResourceUsageReport getResourceUsageReport() { Resource effCap = ((AbstractCSQueue) getQueue()) .getEffectiveCapacity(getAppAMNodePartitionName()); float queueUsagePerc = calc.divide(totalPartitionRes, - report.getUsedResources(), effCap) * 100; + report.getGuaranteedResourcesUsed(), effCap) * 100; report.setQueueUsagePercentage(queueUsagePerc); } return report; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 8a89f78..d2e24bb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -169,8 +169,13 @@ void containerCompleted(RMContainer rmContainer, queue.getMetrics().releaseResources( rmContainer.getNodeLabelExpression(), getUser(), 1, containerResource); - this.attemptResourceUsage.decUsed(containerResource); - getQueue().decUsedGuaranteedResource(containerResource); + if (rmContainer.getExecutionType().equals(ExecutionType.GUARANTEED)) { + this.attemptResourceUsage.decUsed(containerResource); + getQueue().decUsedGuaranteedResource(containerResource); + } else { + this.attemptOpportunisticResourceUsage.decUsed(containerResource); + } + // Clear resource utilization metrics cache. lastMemoryAggregateAllocationUpdateTime = -1; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index ac88f86..14dab9f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -95,6 +95,10 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, th(".runningcontainer", "Running Containers"). th(".allocatedCpu", "Allocated CPU VCores"). th(".allocatedMemory", "Allocated Memory MB"). + th(".allocatedOpportunisticCpu", + "Allocated Opportunistic CPU Vcores"). + th(".allocatedOpportunisticMemory", + "Allocated Opportunistic Memory MB"). th(".reservedCpu", "Reserved CPU VCores"). th(".reservedMemory", "Reserved Memory MB"). th(".progress", "Progress"). @@ -144,6 +148,10 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, .valueOf(appInfo.getAllocatedVCores())).append("\",\"") .append(appInfo.getAllocatedMB() == -1 ? "N/A" : String .valueOf(appInfo.getAllocatedMB())).append("\",\"") + .append(appInfo.getAllocatedOpportunisticVCores() == -1 ? "N/A" : String + .valueOf(appInfo.getAllocatedOpportunisticVCores())).append("\",\"") + .append(appInfo.getAllocatedOpportunisticMB() == -1 ? "N/A" : String + .valueOf(appInfo.getAllocatedOpportunisticMB())).append("\",\"") .append(appInfo.getReservedVCores() == -1 ? "N/A" : String .valueOf(appInfo.getReservedVCores())).append("\",\"") .append(appInfo.getReservedMB() == -1 ? "N/A" : String diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java index 8553d8c..6cd10f3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java @@ -107,9 +107,12 @@ protected void createApplicationMetricsTable(Block html){ attemptResourcePreempted) .__("Number of Non-AM Containers Preempted from Current Attempt:", attemptNumNonAMContainerPreempted) - .__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" : - StringHelper - .getResourceSecondsString(appMetrics.getResourceSecondsMap())) + .__("Aggregate Guaranteed Resource Allocation:", appMetrics == null ? + "N/A" : StringHelper.getResourceSecondsString( + appMetrics.getGuaranteedResourceSecondsMap())) + .__("Aggregate Opportunistic Resource Allocation:", appMetrics == null ? + "N/A" : StringHelper.getResourceSecondsString( + appMetrics.getOpportunisticResourceSecondsMap())) .__("Aggregate Preempted Resource Allocation:", appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString( appMetrics.getPreemptedResourceSecondsMap())); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index d0dccab..0510d62 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -69,6 +69,10 @@ protected void renderData(Block html) { .th(".allocatedMemory", "Allocated Memory MB") .th(".reservedCpu", "Reserved CPU VCores") .th(".reservedMemory", "Reserved Memory MB") + .th(".allocatedOpportunisticCpu", + "Allocated Opportunistic CPU VCores") + .th(".allocatedOpportunisticMemory", + "Allocated Opportunistic Memory MB") .th(".queuePercentage", "% of Queue") .th(".clusterPercentage", "% of Cluster") .th(".progress", "Progress") @@ -155,6 +159,12 @@ protected void renderData(Block html) { .append(app.getReservedMemoryMB() == -1 ? "N/A" : String.valueOf(app.getReservedMemoryMB())) .append("\",\"") + .append(app.getAllocatedOpportunisticCpuVcores() == -1 ? "N/A" : String + .valueOf(app.getAllocatedOpportunisticCpuVcores())) + .append("\",\"") + .append(app.getAllocatedOpportunisticMemoryMB() == -1 ? "N/A" : + String.valueOf(app.getAllocatedOpportunisticMemoryMB())) + .append("\",\"") .append(queuePercent) .append("\",\"") .append(clusterPercent) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 0711b45..669caf9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -94,6 +94,8 @@ private String amRPCAddress; private long allocatedMB; private long allocatedVCores; + private long allocatedOpportunisticMB; + private long allocatedOpportunisticVCores; private long reservedMB; private long reservedVCores; private int runningContainers; @@ -102,6 +104,9 @@ protected float queueUsagePercentage; protected float clusterUsagePercentage; protected Map resourceSecondsMap; + private long opportunisticMemorySeconds; + private long opportunisticVcoreSeconds; + private Map opportunisticResourceSecondsMap; // preemption info fields private long preemptedResourceMB; @@ -199,10 +204,17 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, ApplicationResourceUsageReport resourceReport = attempt.getApplicationResourceUsageReport(); if (resourceReport != null) { - Resource usedResources = resourceReport.getUsedResources(); + Resource guaranteedResourcesUsed = + resourceReport.getGuaranteedResourcesUsed(); + Resource opportunisticResourceUsed = + resourceReport.getOpportunisticResourcesUsed(); Resource reservedResources = resourceReport.getReservedResources(); - allocatedMB = usedResources.getMemorySize(); - allocatedVCores = usedResources.getVirtualCores(); + allocatedMB = guaranteedResourcesUsed.getMemorySize(); + allocatedVCores = guaranteedResourcesUsed.getVirtualCores(); + allocatedOpportunisticMB = + opportunisticResourceUsed.getMemorySize(); + allocatedOpportunisticVCores = + opportunisticResourceUsed.getVirtualCores(); reservedMB = reservedResources.getMemorySize(); reservedVCores = reservedResources.getVirtualCores(); runningContainers = resourceReport.getNumUsedContainers(); @@ -236,9 +248,13 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.getResourcePreempted().getVirtualCores(); - memorySeconds = appMetrics.getMemorySeconds(); - vcoreSeconds = appMetrics.getVcoreSeconds(); - resourceSecondsMap = appMetrics.getResourceSecondsMap(); + memorySeconds = appMetrics.getGuaranteedMemorySeconds(); + vcoreSeconds = appMetrics.getGuaranteedVcoreSeconds(); + resourceSecondsMap = appMetrics.getGuaranteedResourceSecondsMap(); + opportunisticMemorySeconds = appMetrics.getOpportunisticMemorySeconds(); + opportunisticVcoreSeconds = appMetrics.getOpportunisticVcoreSeconds(); + opportunisticResourceSecondsMap = + appMetrics.getOpportunisticResourceSecondsMap(); preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds(); preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap(); @@ -451,6 +467,13 @@ public long getAllocatedVCores() { return this.allocatedVCores; } + public long getAllocatedOpportunisticMB() { + return this.allocatedOpportunisticMB; + } + + public long getAllocatedOpportunisticVCores() { + return this.allocatedOpportunisticVCores; + } public long getReservedMB() { return this.reservedMB; } @@ -475,18 +498,30 @@ public int getNumAMContainersPreempted() { return numAMContainerPreempted; } - public long getMemorySeconds() { + public long getGuaranteedMemorySeconds() { return memorySeconds; } - public long getVcoreSeconds() { + public long getGuaranteedVcoreSeconds() { return vcoreSeconds; } - public Map getResourceSecondsMap() { + public long getOpportunisticMemorySeconds() { + return opportunisticMemorySeconds; + } + + public long getOpportunisticVcoreSeconds() { + return opportunisticVcoreSeconds; + } + + public Map getGuaranteedResourceSecondsMap() { return resourceSecondsMap; } + public Map getOpportunisticResourceSecondsMap() { + return opportunisticResourceSecondsMap; + } + public long getPreemptedMemorySeconds() { return preemptedMemorySeconds; } @@ -579,6 +614,15 @@ public void setAllocatedVCores(long allocatedVCores) { this.allocatedVCores = allocatedVCores; } + public void setAllocatedOpportunisticMB(long allocatedOpportunisticMB) { + this.allocatedOpportunisticMB = allocatedOpportunisticMB; + } + + public void setAllocatedOpportunisticVCores( + long allocatedOpportunisticVCores) { + this.allocatedOpportunisticVCores = allocatedOpportunisticVCores; + } + public void setReservedMB(long reservedMB) { this.reservedMB = reservedMB; } @@ -591,14 +635,22 @@ public void setRunningContainers(int runningContainers) { this.runningContainers = runningContainers; } - public void setMemorySeconds(long memorySeconds) { + public void setGuaranteedMemorySeconds(long memorySeconds) { this.memorySeconds = memorySeconds; } - public void setVcoreSeconds(long vcoreSeconds) { + public void setGuaranteedVcoreSeconds(long vcoreSeconds) { this.vcoreSeconds = vcoreSeconds; } + public void setOpportunisticMemorySeconds(long memorySeconds) { + this.opportunisticMemorySeconds = memorySeconds; + } + + public void setOpportunisticVcoreSeconds(long vcoreSeconds) { + this.opportunisticVcoreSeconds = vcoreSeconds; + } + public void setAppId(String appId) { this.id = appId; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 39a56a8..1721b1a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -89,6 +89,7 @@ message ApplicationAttemptStateDataProto { optional int64 preempted_vcore_seconds = 14; repeated StringLongMapProto application_resource_usage_map = 15; repeated StringLongMapProto preempted_resource_usage_map = 16; + repeated StringLongMapProto application_opportunistic_resource_usage_map = 17; } message EpochProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 140c15f..6ad6006 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -1031,12 +1031,20 @@ public void testEscapeApplicationSummary() { when(app.getState()).thenReturn(RMAppState.RUNNING); when(app.getApplicationType()).thenReturn("MAPREDUCE"); when(app.getSubmitTime()).thenReturn(1000L); - Map resourceSecondsMap = new HashMap<>(); - resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L); - resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L); + Map guaranteedResourceSecondsMap = new HashMap<>(); + guaranteedResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 16384L); + guaranteedResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 64L); + Map opportunisticResourceSecondsMap = new HashMap<>(); + opportunisticResourceSecondsMap.put( + ResourceInformation.MEMORY_MB.getName(), 16384L); + opportunisticResourceSecondsMap.put( + ResourceInformation.VCORES.getName(), 64L); RMAppMetrics metrics = new RMAppMetrics(Resource.newInstance(1234, 56), - 10, 1, resourceSecondsMap, new HashMap<>()); + 10, 1, guaranteedResourceSecondsMap, + new HashMap<>(), opportunisticResourceSecondsMap); when(app.getRMAppMetrics()).thenReturn(metrics); RMAppManager.ApplicationSummary.SummaryBuilder summary = @@ -1047,16 +1055,19 @@ public void testEscapeApplicationSummary() { Assert.assertFalse(msg.contains("\r")); String escaped = "\\n\\n\\r\\r"; - assertTrue(msg.contains("Multiline" + escaped +"AppName")); - assertTrue(msg.contains("Multiline" + escaped +"UserName")); - assertTrue(msg.contains("Multiline" + escaped +"QueueName")); - assertTrue(msg.contains("submitTime=1000")); - assertTrue(msg.contains("memorySeconds=16384")); - assertTrue(msg.contains("vcoreSeconds=64")); - assertTrue(msg.contains("preemptedAMContainers=1")); - assertTrue(msg.contains("preemptedNonAMContainers=10")); - assertTrue(msg.contains("preemptedResources=")); - assertTrue(msg.contains("applicationType=MAPREDUCE")); + Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName")); + Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName")); + Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName")); + Assert.assertTrue(msg.contains("submitTime=1000")); + Assert.assertTrue(msg.contains("guaranteedMemorySeconds=16384")); + Assert.assertTrue(msg.contains("guaranteedVcoreSeconds=64")); + Assert.assertTrue(msg.contains("opportunisticMemorySeconds=16384")); + Assert.assertTrue(msg.contains("opportunisticVcoreSeconds=64")); + Assert.assertTrue(msg.contains("preemptedAMContainers=1")); + Assert.assertTrue(msg.contains("preemptedNonAMContainers=10")); + Assert.assertTrue( + msg.contains("preemptedResources=")); + Assert.assertTrue(msg.contains("applicationType=MAPREDUCE")); } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index c7ed02c..698ca90 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -426,9 +426,11 @@ private void verifyEnemyAppReport(ApplicationReport appReport) { Assert.assertEquals("Enemy should not see app reserved containers", -1, usageReport.getNumReservedContainers()); Assert.assertEquals("Enemy should not see app used resources", - -1, usageReport.getUsedResources().getMemorySize()); + -1, usageReport.getGuaranteedResourcesUsed().getMemorySize()); Assert.assertEquals("Enemy should not see app reserved resources", -1, usageReport.getReservedResources().getMemorySize()); + Assert.assertEquals("Enemy should not see app used opportunistic resources", + -1, usageReport.getOpportunisticResourcesUsed().getMemorySize()); Assert.assertEquals("Enemy should not see app needed resources", -1, usageReport.getNeededResources().getMemorySize()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 1c50dd3..ae8e26b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -370,8 +370,8 @@ public void testGetApplicationReport() throws Exception { ApplicationReport report = response.getApplicationReport(); ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport(); - Assert.assertEquals(10, usageReport.getMemorySeconds()); - Assert.assertEquals(3, usageReport.getVcoreSeconds()); + Assert.assertEquals(10, usageReport.getGuaranteedMemorySeconds()); + Assert.assertEquals(3, usageReport.getGuaranteedVcoreSeconds()); Assert.assertEquals("", report.getAmNodeLabelExpression()); Assert.assertEquals("", report.getAppNodeLabelExpression()); @@ -1416,8 +1416,8 @@ public ApplicationReport createAndGetApplicationReport( clientUserName, allowAccess); ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport(); - usageReport.setMemorySeconds(memorySeconds); - usageReport.setVcoreSeconds(vcoreSeconds); + usageReport.setGuaranteedMemorySeconds(memorySeconds); + usageReport.setGuaranteedVcoreSeconds(vcoreSeconds); report.setApplicationResourceUsageReport(usageReport); return report; } @@ -1474,7 +1474,8 @@ private static YarnScheduler mockYarnScheduler() throws YarnException { when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn( Arrays.asList(getApplicationAttemptId(103))); ApplicationAttemptId attemptId = getApplicationAttemptId(1); - when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null); + when(yarnScheduler.getAppActiveResourceUsageReport(attemptId)) + .thenReturn(null); ResourceCalculator rs = mock(ResourceCalculator.class); when(yarnScheduler.getResourceCalculator()).thenReturn(rs); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java index 3508ab4..66ffdc6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java @@ -23,17 +23,13 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; -import java.util.Map; -import org.apache.commons.lang.time.DateUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -43,6 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.log4j.Level; @@ -84,13 +81,13 @@ public void testUsageWithOneAttemptAndOneContainer() throws Exception { RMAppMetrics rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( - "Before app submittion, memory seconds should have been 0 but was " - + rmAppMetrics.getMemorySeconds(), - rmAppMetrics.getMemorySeconds() == 0); + "Before app submission, memory seconds should have been 0 but was " + + rmAppMetrics.getGuaranteedMemorySeconds(), + rmAppMetrics.getGuaranteedMemorySeconds() == 0); Assert.assertTrue( "Before app submission, vcore seconds should have been 0 but was " - + rmAppMetrics.getVcoreSeconds(), - rmAppMetrics.getVcoreSeconds() == 0); + + rmAppMetrics.getGuaranteedVcoreSeconds(), + rmAppMetrics.getGuaranteedVcoreSeconds() == 0); RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); @@ -105,7 +102,8 @@ public void testUsageWithOneAttemptAndOneContainer() throws Exception { // Allow metrics to accumulate. int sleepInterval = 1000; int cumulativeSleepTime = 0; - while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) { + while (rmAppMetrics.getGuaranteedMemorySeconds() <= 0 + && cumulativeSleepTime < 5000) { Thread.sleep(sleepInterval); cumulativeSleepTime += sleepInterval; } @@ -113,27 +111,35 @@ public void testUsageWithOneAttemptAndOneContainer() throws Exception { rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( "While app is running, memory seconds should be >0 but is " - + rmAppMetrics.getMemorySeconds(), - rmAppMetrics.getMemorySeconds() > 0); + + rmAppMetrics.getGuaranteedMemorySeconds(), + rmAppMetrics.getGuaranteedMemorySeconds() > 0); Assert.assertTrue( "While app is running, vcore seconds should be >0 but is " - + rmAppMetrics.getVcoreSeconds(), - rmAppMetrics.getVcoreSeconds() > 0); + + rmAppMetrics.getGuaranteedVcoreSeconds(), + rmAppMetrics.getGuaranteedVcoreSeconds() > 0); MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer); rmAppMetrics = app0.getRMAppMetrics(); - Assert.assertEquals("Unexpected MemorySeconds value", - ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds()); - Assert.assertEquals("Unexpected VcoreSeconds value", - ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds()); + Assert.assertEquals("Unexpected GuaranteedMemorySeconds value", + ru.getGuaranteedMemorySeconds(), + rmAppMetrics.getGuaranteedMemorySeconds()); + Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value", + ru.getGuaranteedVcoreSeconds(), + rmAppMetrics.getGuaranteedVcoreSeconds()); + Assert.assertEquals("Unexpected OpportunisticMemorySeconds value", + ru.getOpportunisticMemorySeconds(), + rmAppMetrics.getOpportunisticMemorySeconds()); + Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value", + ru.getOpportunisticVcoreSeconds(), + rmAppMetrics.getOpportunisticVcoreSeconds()); rm.stop(); } - @Test (timeout = 120000) + @Test () public void testUsageWithMultipleContainersAndRMRestart() throws Exception { // Set max attempts to 1 so that when the first attempt fails, the app // won't try to start a new one. @@ -191,7 +197,7 @@ public void testUsageWithMultipleContainersAndRMRestart() throws Exception { // Allow metrics to accumulate. int sleepInterval = 1000; int cumulativeSleepTime = 0; - while (app0.getRMAppMetrics().getMemorySeconds() <= 0 + while (app0.getRMAppMetrics().getGuaranteedMemorySeconds() <= 0 && cumulativeSleepTime < 5000) { Thread.sleep(sleepInterval); cumulativeSleepTime += sleepInterval; @@ -216,19 +222,29 @@ public void testUsageWithMultipleContainersAndRMRestart() throws Exception { rm0.waitForState(nm, cId, RMContainerState.COMPLETED); // Check that the container metrics match those from the app usage report. - long memorySeconds = 0; - long vcoreSeconds = 0; + long guaranteedMemorySeconds = 0; + long guaranteedVcoreSeconds = 0; + long opportunisticMemorySeconds = 0; + long opportunisticVcoreSeconds = 0; for (RMContainer c : rmContainers) { AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); - memorySeconds += ru.getMemorySeconds(); - vcoreSeconds += ru.getVcoreSeconds(); + guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds(); + guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds(); + opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds(); + opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds(); } RMAppMetrics metricsBefore = app0.getRMAppMetrics(); - Assert.assertEquals("Unexpected MemorySeconds value", - memorySeconds, metricsBefore.getMemorySeconds()); - Assert.assertEquals("Unexpected VcoreSeconds value", - vcoreSeconds, metricsBefore.getVcoreSeconds()); + Assert.assertEquals("Unexpected GuaranteedMemorySeconds value", + guaranteedMemorySeconds, metricsBefore.getGuaranteedMemorySeconds()); + Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value", + guaranteedVcoreSeconds, metricsBefore.getGuaranteedVcoreSeconds()); + Assert.assertEquals("Unexpected OpportunisticMemorySeconds value", + opportunisticMemorySeconds, + metricsBefore.getOpportunisticMemorySeconds()); + Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value", + opportunisticVcoreSeconds, + metricsBefore.getOpportunisticVcoreSeconds()); // create new RM to represent RM restart. Load up the state store. MockRM rm1 = new MockRM(conf, memStore); @@ -238,10 +254,22 @@ public void testUsageWithMultipleContainersAndRMRestart() throws Exception { // Compare container resource usage metrics from before and after restart. RMAppMetrics metricsAfter = app0After.getRMAppMetrics(); - Assert.assertEquals("Vcore seconds were not the same after RM Restart", - metricsBefore.getVcoreSeconds(), metricsAfter.getVcoreSeconds()); - Assert.assertEquals("Memory seconds were not the same after RM Restart", - metricsBefore.getMemorySeconds(), metricsAfter.getMemorySeconds()); + Assert.assertEquals( + "Guaranteed vcore seconds were not the same after RM Restart", + metricsBefore.getGuaranteedVcoreSeconds(), + metricsAfter.getGuaranteedVcoreSeconds()); + Assert.assertEquals( + "Guaranteed memory seconds were not the same after RM Restart", + metricsBefore.getGuaranteedMemorySeconds(), + metricsAfter.getGuaranteedMemorySeconds()); + Assert.assertEquals( + "Opportunistic vcore seconds were not the same after RM Restart", + metricsBefore.getOpportunisticVcoreSeconds(), + metricsAfter.getOpportunisticVcoreSeconds()); + Assert.assertEquals( + "Opportunistic memory seconds were not the same after RM Restart", + metricsBefore.getOpportunisticMemorySeconds(), + metricsAfter.getOpportunisticMemorySeconds()); rm0.stop(); rm0.close(); @@ -312,8 +340,10 @@ private void amRestartTests(boolean keepRunningContainers) amContainerId.getContainerId(), ContainerState.COMPLETE); rm.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FAILED); rm.drainEvents(); - long memorySeconds = 0; - long vcoreSeconds = 0; + long guaranteedMemorySeconds = 0; + long guaranteedVcoreSeconds = 0; + long opportunisticMemorySeconds = 0; + long opportunisticVcoreSeconds = 0; // Calculate container usage metrics for first attempt. if (keepRunningContainers) { @@ -321,8 +351,10 @@ private void amRestartTests(boolean keepRunningContainers) for (RMContainer c : rmContainers) { if (c.getContainerId().equals(amContainerId)) { AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); - memorySeconds += ru.getMemorySeconds(); - vcoreSeconds += ru.getVcoreSeconds(); + guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds(); + guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds(); + opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds(); + opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds(); } else { // The remaining container should be RUNNING. Assert.assertTrue("After first attempt failed, remaining container " @@ -336,8 +368,10 @@ private void amRestartTests(boolean keepRunningContainers) for (RMContainer c : rmContainers) { waitforContainerCompletion(rm, nm, amContainerId, c); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); - memorySeconds += ru.getMemorySeconds(); - vcoreSeconds += ru.getVcoreSeconds(); + guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds(); + guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds(); + opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds(); + opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds(); } } @@ -388,16 +422,24 @@ private void amRestartTests(boolean keepRunningContainers) for (RMContainer c : rmContainers) { waitforContainerCompletion(rm, nm, amContainerId, c); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); - memorySeconds += ru.getMemorySeconds(); - vcoreSeconds += ru.getVcoreSeconds(); + guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds(); + guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds(); + opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds(); + opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds(); } RMAppMetrics rmAppMetrics = app.getRMAppMetrics(); - Assert.assertEquals("Unexpected MemorySeconds value", - memorySeconds, rmAppMetrics.getMemorySeconds()); - Assert.assertEquals("Unexpected VcoreSeconds value", - vcoreSeconds, rmAppMetrics.getVcoreSeconds()); + Assert.assertEquals("Unexpected GuaranteedMemorySeconds value", + guaranteedMemorySeconds, rmAppMetrics.getGuaranteedMemorySeconds()); + Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value", + guaranteedVcoreSeconds, rmAppMetrics.getGuaranteedVcoreSeconds()); + Assert.assertEquals("Unexpected OpportunisticMemorySeconds value", + opportunisticMemorySeconds, + rmAppMetrics.getOpportunisticMemorySeconds()); + Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value", + opportunisticVcoreSeconds, + rmAppMetrics.getOpportunisticVcoreSeconds()); rm.stop(); return; @@ -419,16 +461,10 @@ private void waitforContainerCompletion(MockRM rm, MockNM nm, private AggregateAppResourceUsage calculateContainerResourceMetrics( RMContainer rmContainer) { - Resource resource = rmContainer.getContainer().getResource(); - long usedMillis = - rmContainer.getFinishTime() - rmContainer.getCreationTime(); - long memorySeconds = resource.getMemorySize() - * usedMillis / DateUtils.MILLIS_PER_SECOND; - long vcoreSeconds = resource.getVirtualCores() - * usedMillis / DateUtils.MILLIS_PER_SECOND; - Map map = new HashMap<>(); - map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); - map.put(ResourceInformation.VCORES.getName(), vcoreSeconds); - return new AggregateAppResourceUsage(map); + ContainerResourceUsageReport resourceUsageReport = + rmContainer.getResourceUsageReport(); + return new AggregateAppResourceUsage( + resourceUsageReport.getGuaranteedResourceUsageSecondsMap(), + resourceUsageReport.getOpportunisticResourceSecondsMap()); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 2aca375..f7eb0a5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -193,7 +193,7 @@ public YarnApplicationState createApplicationState() { @Override public RMAppMetrics getRMAppMetrics() { return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(), - new HashMap<>()); + new HashMap<>(), new HashMap<>()); } @Override @@ -348,7 +348,7 @@ public ApplicationReport createAndGetApplicationReport( ApplicationResourceUsageReport usageReport = ApplicationResourceUsageReport .newInstance(0, 0, null, null, null, new HashMap<>(), 0, 0, - new HashMap<>()); + new HashMap<>(), null, new HashMap<>()); ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 2287617..c9e2d04 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -219,13 +219,25 @@ public void testPublishApplicationMetrics() throws Exception { entity.getOtherInfo().get( ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); Assert.assertEquals( - app.getRMAppMetrics().getMemorySeconds(), + app.getRMAppMetrics().getGuaranteedMemorySeconds(), Long.parseLong(entity.getOtherInfo() - .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString())); + .get(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS) + .toString())); + Assert.assertEquals( + app.getRMAppMetrics().getGuaranteedVcoreSeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS) + .toString())); Assert.assertEquals( - app.getRMAppMetrics().getVcoreSeconds(), + app.getRMAppMetrics().getOpportunisticMemorySeconds(), Long.parseLong(entity.getOtherInfo() - .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString())); + .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS) + .toString())); + Assert.assertEquals( + app.getRMAppMetrics().getOpportunisticVcoreSeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS) + .toString())); Assert.assertEquals( app.getRMAppMetrics().getPreemptedMemorySeconds(), Long.parseLong(entity.getOtherInfo() @@ -517,7 +529,8 @@ private static RMApp createRMApp(ApplicationId appId) { .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); when(app.getRMAppMetrics()) - .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap)); + .thenReturn(new RMAppMetrics(null, 0, 0, + resourceMap, preemptedMap, new HashMap<>())); Set appTags = new HashSet(); appTags.add("test"); appTags.add("tags"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 68bb325..7aa1a60 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -221,7 +221,7 @@ public void testPublishApplicationMetrics() throws Exception { File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); verifyEntity( - appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0); + appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 10, 0); } @Test(timeout = 10000) @@ -365,14 +365,14 @@ private static RMApp createRMApp(ApplicationId appId) { when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()) .thenReturn(FinalApplicationStatus.UNDEFINED); - Map resourceSecondsMap = new HashMap<>(); - resourceSecondsMap + Map guaranteedResourceSecondsMap = new HashMap<>(); + guaranteedResourceSecondsMap .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); - resourceSecondsMap + guaranteedResourceSecondsMap .put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); when(app.getRMAppMetrics()).thenReturn( - new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap, - new HashMap<>())); + new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, + guaranteedResourceSecondsMap, new HashMap<>(), new HashMap<>())); when(app.getApplicationTags()).thenReturn(Collections.emptySet()); ApplicationSubmissionContext appSubmissionContext = mock(ApplicationSubmissionContext.class); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 453d805..98199a3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -194,7 +194,8 @@ protected RMAppAttempt storeAttempt(RMStateStore store, when(mockAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(new HashMap<>())); + .thenReturn(new AggregateAppResourceUsage( + new HashMap<>(), new HashMap<>())); dispatcher.attemptId = attemptId; store.storeNewApplicationAttempt(mockAttempt); waitNotify(dispatcher); @@ -292,7 +293,8 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, when(mockRemovedAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(new HashMap<>())); + .thenReturn(new AggregateAppResourceUsage( + new HashMap<>(), new HashMap<>())); attempts.put(attemptIdRemoved, mockRemovedAttempt); store.removeApplication(mockRemovedApp); @@ -369,7 +371,8 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 100, - oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>()); + oldAttemptState.getFinishTime(), new HashMap<>(), + new HashMap<>(), new HashMap<>()); store.updateApplicationAttemptState(newAttemptState); // test updating the state of an app/attempt whose initial state was not @@ -393,7 +396,8 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 111, - oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>()); + oldAttemptState.getFinishTime(), new HashMap<>(), + new HashMap<>(), new HashMap<>()); store.updateApplicationAttemptState(dummyAttempt); // let things settle down diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 6a8f47d..4d2c2da 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -543,7 +543,8 @@ public void testFencedState() throws Exception { when(mockAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(new HashMap<>())); + .thenReturn(new AggregateAppResourceUsage(new HashMap<>(), + new HashMap<>(0))); store.storeNewApplicationAttempt(mockAttempt); assertEquals("RMStateStore should have been in fenced state", true, store.isFencedState()); @@ -555,7 +556,7 @@ public void testFencedState() throws Exception { store.getCredentialsFromAppAttempt(mockAttempt), startTime, RMAppAttemptState.FINISHED, "testUrl", "test", FinalApplicationStatus.SUCCEEDED, 100, - finishTime, new HashMap<>(), new HashMap<>()); + finishTime, new HashMap<>(), new HashMap<>(), new HashMap<>()); store.updateApplicationAttemptState(newAttemptState); assertEquals("RMStateStore should have been in fenced state", true, store.isFencedState()); @@ -792,20 +793,33 @@ private static ApplicationStateData createAppState( private static ApplicationAttemptStateData createFinishedAttempt( ApplicationAttemptId attemptId, Container container, long startTime, int amExitStatus) { - Map resourceSecondsMap = new HashMap<>(); - Map preemptedResoureSecondsMap = new HashMap<>(); - resourceSecondsMap - .put(ResourceInformation.MEMORY_MB.getName(), 0L); - resourceSecondsMap - .put(ResourceInformation.VCORES.getName(), 0L); - preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), - 0L); - preemptedResoureSecondsMap - .put(ResourceInformation.VCORES.getName(), 0L); + Map guaranteedResourceSecondsMap = + new HashMap() { + { + put(ResourceInformation.MEMORY_MB.getName(), 0L); + put(ResourceInformation.VCORES.getName(), 0L); + } + }; + Map opportunisticResourceSecondsMap = + new HashMap() { + { + put(ResourceInformation.MEMORY_MB.getName(), 0L); + put(ResourceInformation.VCORES.getName(), 0L); + } + }; + Map preemptedResoureSecondsMap = + new HashMap() { + { + put(ResourceInformation.MEMORY_MB.getName(), 0L); + put(ResourceInformation.VCORES.getName(), 0L); + } + }; + return ApplicationAttemptStateData.newInstance(attemptId, container, null, startTime, RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, - amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap); + amExitStatus, 0, guaranteedResourceSecondsMap, + preemptedResoureSecondsMap, opportunisticResourceSecondsMap); } private ApplicationAttemptId storeAttempt(RMStateStore store, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 344f8bb7..37fc7fc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -54,7 +54,6 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -243,7 +242,7 @@ public void setUp() throws Exception { ResourceScheduler resourceScheduler = mock(ResourceScheduler.class); doReturn(null).when(resourceScheduler) - .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()); + .getAppActiveResourceUsageReport(Matchers.any()); doReturn(resourceScheduler).when(rmContext).getScheduler(); doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext) @@ -1214,7 +1213,8 @@ public void testGetAppReport() throws IOException { assertAppState(RMAppState.NEW, app); ApplicationReport report = app.createAndGetApplicationReport(null, true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); - Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT); + Assert.assertEquals(RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT, + report.getApplicationResourceUsageReport()); report = app.createAndGetApplicationReport("clientuser", true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); Assert.assertTrue("bad proxy url for app", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 9dd5703..b53e9d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -299,10 +299,12 @@ public void setUp() throws Exception { ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class); - when(appResUsgRpt.getMemorySeconds()).thenReturn(0L); - when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L); + when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(0L); + when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(0L); + when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(0L); + when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(0L); when(resourceScheduler - .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any())) + .getAppActiveResourceUsageReport((ApplicationAttemptId)Matchers.any())) .thenReturn(appResUsgRpt); spyRMContext = spy(rmContext); Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler(); @@ -754,10 +756,12 @@ public void testUsageReport() { ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId(); ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class); - when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L); - when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L); - when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class))) - .thenReturn(appResUsgRpt); + when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(123456L); + when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(55544L); + when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(123456L); + when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(55544L); + when(scheduler.getAppActiveResourceUsageReport( + any(ApplicationAttemptId.class))).thenReturn(appResUsgRpt); // start and finish the attempt Container amContainer = allocateApplicationAttempt(); @@ -769,12 +773,14 @@ public void testUsageReport() { // expect usage stats to come from the scheduler report ApplicationResourceUsageReport report = applicationAttempt.getApplicationResourceUsageReport(); - Assert.assertEquals(123456L, report.getMemorySeconds()); - Assert.assertEquals(55544L, report.getVcoreSeconds()); + Assert.assertEquals(123456L, report.getGuaranteedMemorySeconds()); + Assert.assertEquals(55544L, report.getGuaranteedVcoreSeconds()); + Assert.assertEquals(123456L, report.getOpportunisticMemorySeconds()); + Assert.assertEquals(55544L, report.getOpportunisticVcoreSeconds()); // finish app attempt and remove it from scheduler - when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L); - when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L); + when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(223456L); + when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(75544L); sendAttemptUpdateSavedEvent(applicationAttempt); NodeId anyNodeId = NodeId.newInstance("host", 1234); applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( @@ -785,8 +791,8 @@ public void testUsageReport() { when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null); report = applicationAttempt.getApplicationResourceUsageReport(); - Assert.assertEquals(223456, report.getMemorySeconds()); - Assert.assertEquals(75544, report.getVcoreSeconds()); + Assert.assertEquals(223456, report.getGuaranteedMemorySeconds()); + Assert.assertEquals(75544, report.getGuaranteedVcoreSeconds()); } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index cd9d211..f31fb1f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -103,7 +103,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; -import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; @@ -2858,7 +2857,7 @@ public void testAppReservationWithDominantResourceCalculator() throws Exception Thread.sleep(1000); AllocateResponse allocResponse = am1.schedule(); ApplicationResourceUsageReport report = - rm.getResourceScheduler().getAppResourceUsageReport( + rm.getResourceScheduler().getAppActiveResourceUsageReport( attempt1.getAppAttemptId()); Assert.assertEquals(0, allocResponse.getAllocatedContainers().size()); Assert.assertEquals(0, report.getNumReservedContainers()); @@ -2871,7 +2870,7 @@ public void testAppReservationWithDominantResourceCalculator() throws Exception allocResponse = am1.schedule(); } report = - rm.getResourceScheduler().getAppResourceUsageReport( + rm.getResourceScheduler().getAppActiveResourceUsageReport( attempt1.getAppAttemptId()); Assert.assertEquals(1, allocResponse.getAllocatedContainers().size()); Assert.assertEquals(0, report.getNumReservedContainers()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index b4ebd15..a3ae7bf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -223,23 +223,23 @@ public RMNodeLabelsManager createNodeLabelManager() { RMApp app1 = rm.submitApp(GB, "app", "user", null, "a"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); ApplicationResourceUsageReport appResourceUsageReport = - rm.getResourceScheduler().getAppResourceUsageReport( + rm.getResourceScheduler().getAppActiveResourceUsageReport( am1.getApplicationAttemptId()); - Assert.assertEquals(1024, appResourceUsageReport.getUsedResources() - .getMemorySize()); - Assert.assertEquals(1, appResourceUsageReport.getUsedResources() - .getVirtualCores()); + Assert.assertEquals(1024, appResourceUsageReport + .getGuaranteedResourcesUsed().getMemorySize()); + Assert.assertEquals(1, appResourceUsageReport + .getGuaranteedResourcesUsed().getVirtualCores()); // request a container. am1.allocate("*", GB, 1, new ArrayList(), "x"); containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED); appResourceUsageReport = - rm.getResourceScheduler().getAppResourceUsageReport( + rm.getResourceScheduler().getAppActiveResourceUsageReport( am1.getApplicationAttemptId()); - Assert.assertEquals(2048, appResourceUsageReport.getUsedResources() - .getMemorySize()); - Assert.assertEquals(2, appResourceUsageReport.getUsedResources() - .getVirtualCores()); + Assert.assertEquals(2048, appResourceUsageReport + .getGuaranteedResourcesUsed().getMemorySize()); + Assert.assertEquals(2, appResourceUsageReport + .getGuaranteedResourcesUsed().getVirtualCores()); LeafQueue queue = (LeafQueue) ((CapacityScheduler) rm.getResourceScheduler()) .getQueue("a"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java index f88ac8b..de97ea4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java @@ -66,7 +66,7 @@ public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception { RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(), - new HashMap<>()); + new HashMap<>(), new HashMap<>()); when(app.getRMAppMetrics()).thenReturn(appMetrics); // initialize RM Context, and create RMApp, without creating RMAppAttempt diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index b125608..93efe23 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Token; @@ -259,7 +258,7 @@ public static ClientRMService mockClientRMService(RMContext rmContext) { app.getDiagnostics().toString(), (String) null, app.getStartTime(), app.getFinishTime(), app.getFinalApplicationStatus(), - (ApplicationResourceUsageReport) null, app.getTrackingUrl(), + null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null); appReports.add(appReport); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java index cea0088..87846c7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java @@ -138,7 +138,7 @@ private static RMContext mockRMContext(List states) { @Override public RMAppMetrics getRMAppMetrics() { return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, - new HashMap<>(), new HashMap<>()); + new HashMap<>(), new HashMap<>(), new HashMap<>()); } @Override public YarnApplicationState createApplicationState() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index f0704ac..8250bba 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1603,7 +1603,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 38 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 43 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index 40bdbd8..6f1ee3d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -339,12 +339,22 @@ private static void mergeAMWithUAM(AppInfo am, AppInfo uam) { am.setAllocatedMB(am.getAllocatedMB() + uam.getAllocatedMB()); am.setAllocatedVCores(am.getAllocatedVCores() + uam.getAllocatedVCores()); + am.setAllocatedOpportunisticMB(am.getAllocatedOpportunisticMB() + + uam.getAllocatedOpportunisticMB()); + am.setAllocatedOpportunisticVCores(am.getAllocatedOpportunisticVCores() + + uam.getAllocatedOpportunisticVCores()); am.setReservedMB(am.getReservedMB() + uam.getReservedMB()); am.setReservedVCores(am.getReservedVCores() + uam.getReservedMB()); am.setRunningContainers( am.getRunningContainers() + uam.getRunningContainers()); - am.setMemorySeconds(am.getMemorySeconds() + uam.getMemorySeconds()); - am.setVcoreSeconds(am.getVcoreSeconds() + uam.getVcoreSeconds()); + am.setGuaranteedMemorySeconds(am.getGuaranteedMemorySeconds() + + uam.getGuaranteedMemorySeconds()); + am.setGuaranteedVcoreSeconds(am.getGuaranteedVcoreSeconds() + + uam.getGuaranteedVcoreSeconds()); + am.setOpportunisticVcoreSeconds(am.getOpportunisticVcoreSeconds() + + uam.getOpportunisticVcoreSeconds()); + am.setOpportunisticMemorySeconds(am.getOpportunisticMemorySeconds() + + uam.getOpportunisticMemorySeconds()); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java index edf3804..1a5c42e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java @@ -249,8 +249,10 @@ public void testMergeAppsRunning() { Assert.assertEquals(value * 3, app.getReservedMB()); Assert.assertEquals(value * 3, app.getReservedVCores()); Assert.assertEquals(value * 3, app.getRunningContainers()); - Assert.assertEquals(value * 3, app.getMemorySeconds()); - Assert.assertEquals(value * 3, app.getVcoreSeconds()); + Assert.assertEquals(value * 3, app.getGuaranteedMemorySeconds()); + Assert.assertEquals(value * 3, app.getGuaranteedVcoreSeconds()); + Assert.assertEquals(value * 3, app.getOpportunisticMemorySeconds()); + Assert.assertEquals(value * 3, app.getOpportunisticVcoreSeconds()); Assert.assertEquals(3, app.getResourceRequests().size()); } @@ -258,11 +260,15 @@ private void setAppInfoRunning(AppInfo am, int value) { am.getResourceRequests().add(new ResourceRequestInfo()); am.setAllocatedMB(value); am.setAllocatedVCores(value); + am.setAllocatedOpportunisticVCores(value); + am.setAllocatedOpportunisticMB(value); am.setReservedMB(value); am.setReservedVCores(value); am.setRunningContainers(value); - am.setMemorySeconds(value); - am.setVcoreSeconds(value); + am.setGuaranteedMemorySeconds(value); + am.setGuaranteedVcoreSeconds(value); + am.setOpportunisticMemorySeconds(value); + am.setOpportunisticVcoreSeconds(value); } /**