From 44684caf4f6accddb8332444d615ae62ead9bc14 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Mon, 10 Jul 2017 22:14:01 +0530 Subject: [PATCH] YARN-6788-YARN-3926.001 --- .../hadoop/yarn/api/records/ProfileCapability.java | 8 +- .../apache/hadoop/yarn/api/records/Resource.java | 141 ++++++++++++--------- .../yarn/api/records/impl/pb/ProtoUtils.java | 5 +- .../yarn/api/records/impl/pb/ResourcePBImpl.java | 93 ++++++++------ .../hadoop/yarn/util/resource/ResourceUtils.java | 34 +++++ .../hadoop/yarn/util/resource/Resources.java | 77 ++++++----- .../rmapp/attempt/RMAppAttemptMetrics.java | 11 +- .../scheduler/SchedulerApplicationAttempt.java | 9 +- .../resourcemanager/webapp/dao/SchedulerInfo.java | 2 +- 9 files changed, 226 insertions(+), 154 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java index 1a8d1c3..2cb4670 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java @@ -162,10 +162,10 @@ public static Resource toResource(ProfileCapability capability, if (capability.getProfileCapabilityOverride() != null && !capability.getProfileCapabilityOverride().equals(none)) { - for (Map.Entry entry : capability - .getProfileCapabilityOverride().getResources().entrySet()) { - if (entry.getValue() != null && entry.getValue().getValue() >= 0) { - resource.setResourceInformation(entry.getKey(), entry.getValue()); + for (ResourceInformation entry : capability + .getProfileCapabilityOverride().getResources()) { + if (entry != null && entry.getValue() >= 0) { + resource.setResourceInformation(entry.getName(), entry); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 9a8e2ec..04fa654 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -70,34 +70,40 @@ SimpleResource(long memory, long vcores) { this.memory = memory; this.vcores = vcores; - } + @Override public int getMemory() { return (int)memory; } + @Override public void setMemory(int memory) { this.memory = memory; } + @Override public long getMemorySize() { return memory; } + @Override public void setMemorySize(long memory) { this.memory = memory; } + @Override public int getVirtualCores() { return (int)vcores; } + @Override public void setVirtualCores(int vcores) { this.vcores = vcores; } + @Override - public Map getResources() { + public ResourceInformation[] getResources() { if (resourceInformationMap == null) { resourceInformationMap = new HashMap<>(); resourceInformationMap.put(ResourceInformation.MEMORY_MB.getName(), @@ -109,14 +115,21 @@ public void setVirtualCores(int vcores) { .setValue(this.memory); resourceInformationMap.get(ResourceInformation.VCORES.getName()) .setValue(this.vcores); - return Collections.unmodifiableMap(resourceInformationMap); + return (ResourceInformation[]) this.resourceInformationMap.values() + .toArray( + new ResourceInformation[this.resourceInformationMap.size()]); + } + + @Override + public Integer getResourceIndex(String resource) { + return null; } } @Public @Stable public static Resource newInstance(int memory, int vCores) { - if (tmpResource.getResources().size() > 2) { + if (tmpResource.getResources().length > 2) { Resource ret = Records.newRecord(Resource.class); ret.setMemorySize(memory); ret.setVirtualCores(vCores); @@ -128,7 +141,7 @@ public static Resource newInstance(int memory, int vCores) { @Public @Stable public static Resource newInstance(long memory, int vCores) { - if (tmpResource.getResources().size() > 2) { + if (tmpResource.getResources().length > 2) { Resource ret = Records.newRecord(Resource.class); ret.setMemorySize(memory); ret.setVirtualCores(vCores); @@ -148,9 +161,8 @@ public static Resource newInstance(Resource resource) { @InterfaceAudience.Private @InterfaceStability.Unstable public static void copy(Resource source, Resource dest) { - for (Map.Entry entry : source.getResources() - .entrySet()) { - dest.setResourceInformation(entry.getKey(), entry.getValue()); + for (ResourceInformation entry : source.getResources()) { + dest.setResourceInformation(entry.getName(), entry); } } @@ -251,7 +263,16 @@ public void setMemorySize(long memory) { */ @Public @Evolving - public abstract Map getResources(); + public abstract ResourceInformation[] getResources(); + + /** + * Get resource index + * @param resource Name of the resource + * @return Resource Index + */ + @Public + @Evolving + public abstract Integer getResourceIndex(String resource); /** * Get ResourceInformation for a specified resource. @@ -264,12 +285,13 @@ public void setMemorySize(long memory) { @Evolving public ResourceInformation getResourceInformation(String resource) throws YarnException { - if (getResources().containsKey(resource)) { - return getResources().get(resource); + Integer index = tmpResource.getResourceIndex(resource); + ResourceInformation[] resources = getResources(); + if (index != null) { + return resources[index]; } - throw new YarnException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + throw new YarnException("Unknown resource '" + resource + + "'. Known resources are " + resources); } /** @@ -283,12 +305,13 @@ public ResourceInformation getResourceInformation(String resource) @Public @Evolving public Long getResourceValue(String resource) throws YarnException { - if (getResources().containsKey(resource)) { - return getResources().get(resource).getValue(); + Integer index = tmpResource.getResourceIndex(resource); + ResourceInformation[] resources = getResources(); + if (index != null) { + return resources[index].getValue(); } - throw new YarnException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + throw new YarnException("Unknown resource '" + resource + + "'. Known resources are " + resources); } /** @@ -301,7 +324,8 @@ public Long getResourceValue(String resource) throws YarnException { @Public @Evolving public void setResourceInformation(String resource, - ResourceInformation resourceInformation) throws ResourceNotFoundException { + ResourceInformation resourceInformation) + throws ResourceNotFoundException { if (resource.equals(ResourceInformation.MEMORY_MB.getName())) { this.setMemorySize(resourceInformation.getValue()); return; @@ -310,14 +334,14 @@ public void setResourceInformation(String resource, this.setVirtualCores((int) resourceInformation.getValue()); return; } - if (getResources().containsKey(resource)) { - ResourceInformation - .copy(resourceInformation, getResources().get(resource)); + Integer index = tmpResource.getResourceIndex(resource); + ResourceInformation[] resources = getResources(); + if (index != null) { + ResourceInformation.copy(resourceInformation, resources[index]); return; } - throw new ResourceNotFoundException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + throw new ResourceNotFoundException("Unknown resource '" + resource + + "'. Known resources are " + resources); } /** @@ -340,13 +364,15 @@ public void setResourceValue(String resource, Long value) this.setVirtualCores(value.intValue()); return; } - if (getResources().containsKey(resource)) { - getResources().get(resource).setValue(value); + + Integer index = tmpResource.getResourceIndex(resource); + ResourceInformation[] resources = getResources(); + if (index != null) { + resources[index].setValue(value); return; } - throw new ResourceNotFoundException( - "Unknown resource '" + resource + "'. Known resources are " - + getResources().keySet()); + throw new ResourceNotFoundException("Unknown resource '" + resource + + "'. Known resources are " + resources); } @Override @@ -356,13 +382,12 @@ public int hashCode() { int result = (int) (939769357 + getMemorySize()); // prime * result = 939769357 initially result = prime * result + getVirtualCores(); - for (Map.Entry entry : getResources() - .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) - || entry.getKey().equals(ResourceInformation.VCORES.getName())) { + for (ResourceInformation entry : getResources()) { + if (entry.getName().equals(ResourceInformation.MEMORY_MB.getName()) + || entry.getName().equals(ResourceInformation.VCORES.getName())) { continue; } - result = prime * result + entry.getValue().hashCode(); + result = prime * result + entry.hashCode(); } return result; } @@ -391,21 +416,20 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append(" entry : getResources() - .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) - && entry.getValue().getUnits() + for (ResourceInformation entry : getResources()) { + if (entry.getName().equals(ResourceInformation.MEMORY_MB.getName()) + && entry.getUnits() .equals(ResourceInformation.MEMORY_MB.getUnits())) { continue; } - if (entry.getKey().equals(ResourceInformation.VCORES.getName()) - && entry.getValue().getUnits() + if (entry.getName().equals(ResourceInformation.VCORES.getName()) + && entry.getUnits() .equals(ResourceInformation.VCORES.getUnits())) { continue; } - sb.append(", ").append(entry.getKey()).append(": ") - .append(entry.getValue().getValue()) - .append(entry.getValue().getUnits()); + sb.append(", ").append(entry.getName()).append(": ") + .append(entry.getValue()) + .append(entry.getUnits()); } sb.append(">"); return sb.toString(); @@ -413,30 +437,33 @@ public String toString() { @Override public int compareTo(Resource other) { - Map thisResources, otherResources; + ResourceInformation[] thisResources, otherResources; thisResources = this.getResources(); otherResources = other.getResources(); - long diff = thisResources.size() - otherResources.size(); + long diff = thisResources.length - otherResources.length; if (diff == 0) { // compare memory and vcores first(in that order) to preserve // existing behaviour - if (thisResources.keySet().equals(otherResources.keySet())) { + if (thisResources.equals(otherResources)) { diff = this.getMemorySize() - other.getMemorySize(); if (diff == 0) { diff = this.getVirtualCores() - other.getVirtualCores(); } if (diff == 0) { - for (Map.Entry entry : thisResources - .entrySet()) { - if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) - || entry.getKey() - .equals(ResourceInformation.VCORES.getName())) { + for (ResourceInformation entry : thisResources) { + if (entry.getName().equals(ResourceInformation.MEMORY_MB.getName()) + || entry.getName() + .equals(ResourceInformation.VCORES.getName())) { continue; } - diff = - entry.getValue().compareTo(otherResources.get(entry.getKey())); - if (diff != 0) { - break; + + for (ResourceInformation otherEntry : otherResources) { + if (entry.getName().equals(otherEntry.getName())) { + diff = entry.compareTo(otherEntry); + if (diff != 0) { + break; + } + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index 626ff9b..158c2ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -456,9 +456,8 @@ public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) { List pList) { Resource tmp = Resource.newInstance(0, 0); Map ret = new HashMap<>(); - for (Map.Entry entry : tmp.getResources() - .entrySet()) { - ret.put(entry.getKey(), 0L); + for (ResourceInformation entry : tmp.getResources()) { + ret.put(entry.getName(), 0L); } if (pList != null) { for (YarnProtos.StringLongMapProto p : pList) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index a9abed9..18b3c3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.api.records.impl.pb; -import org.apache.commons.collections.map.UnmodifiableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -46,9 +45,8 @@ ResourceProto.Builder builder = null; boolean viaProto = false; - private Map resources; - private Map readOnlyResources; - + private ResourceInformation[] resources = null; + private ResourceInformation[] readOnlyResources; // call via ProtoUtils.convertToProtoFormat(Resource) static ResourceProto getProto(Resource r) { @@ -71,7 +69,6 @@ public ResourcePBImpl(ResourceProto proto) { this.proto = proto; viaProto = true; this.readOnlyResources = null; - this.resources = null; initResources(); } @@ -139,7 +136,8 @@ private void initResources() { return; } ResourceProtoOrBuilder p = viaProto ? proto : builder; - initResourcesMap(); + Map indexMap = ResourceUtils.getResourceTypeIndex(); + initResourcesMap(indexMap); for (ResourceInformationProto entry : p.getResourceValueMapList()) { ResourceTypes type = entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) : @@ -148,12 +146,13 @@ private void initResources() { long value = entry.hasValue() ? entry.getValue() : 0L; ResourceInformation ri = ResourceInformation .newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE); - if (resources.containsKey(ri.getName())) { - resources.get(ri.getName()).setResourceType(ri.getResourceType()); - resources.get(ri.getName()).setUnits(ri.getUnits()); - resources.get(ri.getName()).setValue(value); - } else { + Integer index = indexMap.get(entry.getKey()); + if(index == null) { LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping"); + } else { + resources[index].setResourceType(ri.getResourceType()); + resources[index].setUnits(ri.getUnits()); + resources[index].setValue(value); } } this.setMemorySize(p.getMemory()); @@ -172,8 +171,9 @@ public void setResourceInformation(String resource, resourceInformation.setName(resource); } initResources(); - if (resources.containsKey(resource)) { - ResourceInformation.copy(resourceInformation, resources.get(resource)); + Integer index = ResourceUtils.getResourceTypeIndex().get(resource); + if(index != null) { + ResourceInformation.copy(resourceInformation, resources[index]); } } @@ -185,15 +185,17 @@ public void setResourceValue(String resource, Long value) if (resource == null) { throw new IllegalArgumentException("resource type object cannot be null"); } - if (resources == null || (!resources.containsKey(resource))) { + + Integer index = ResourceUtils.getResourceTypeIndex().get(resource); + if (index == null) { throw new ResourceNotFoundException( "Resource " + resource + " not found"); } - resources.get(resource).setValue(value); + resources[index].setValue(value); } @Override - public Map getResources() { + public ResourceInformation[] getResources() { initResources(); return readOnlyResources; } @@ -201,49 +203,52 @@ public void setResourceValue(String resource, Long value) @Override public ResourceInformation getResourceInformation(String resource) { initResources(); - if (this.resources.containsKey(resource)) { - return this.resources.get(resource); + Integer index = ResourceUtils.getResourceTypeIndex().get(resource); + if (index == null) { + throw new ResourceNotFoundException( + "Could not find entry for " + resource); } - throw new ResourceNotFoundException("Could not find entry for " + resource); + return resources[index]; } @Override public Long getResourceValue(String resource) { initResources(); - if (this.resources.containsKey(resource)) { - return this.resources.get(resource).getValue(); + Integer index = ResourceUtils.getResourceTypeIndex().get(resource); + if(index == null) { + throw new ResourceNotFoundException("Could not find entry for " + resource); } - throw new ResourceNotFoundException("Could not find entry for " + resource); + return resources[index].getValue(); } - private void initResourcesMap() { + private void initResourcesMap(Map indexMap) { if (resources == null) { - resources = new HashMap<>(); - Map types = ResourceUtils.getResourceTypes(); + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); if (types == null) { throw new YarnRuntimeException( "Got null return value from ResourceUtils.getResourceTypes()"); } - for (Map.Entry entry : types.entrySet()) { - resources.put(entry.getKey(), - ResourceInformation.newInstance(entry.getValue())); + + resources = new ResourceInformation[types.length]; + for (ResourceInformation entry : types) { + int index = indexMap.get(entry.getName()); + resources[index] = ResourceInformation.newInstance(entry); } - readOnlyResources = Collections.unmodifiableMap(resources); + readOnlyResources = Arrays.copyOf(resources, resources.length); } } synchronized private void mergeLocalToBuilder() { builder.clearResourceValueMap(); - if (resources != null && !resources.isEmpty()) { - for (Map.Entry entry : - resources.entrySet()) { - ResourceInformationProto.Builder e = - ResourceInformationProto.newBuilder(); - e.setKey(entry.getKey()); - e.setUnits(entry.getValue().getUnits()); - e.setType( - ProtoUtils.converToProtoFormat(entry.getValue().getResourceType())); - e.setValue(entry.getValue().getValue()); + if(resources != null && resources.length != 0) { + for (int i = 0; i < resources.length - 1; i++) { + ResourceInformation resInfo = resources[i]; + ResourceInformationProto.Builder e = ResourceInformationProto + .newBuilder(); + e.setKey(resInfo.getName()); + e.setUnits(resInfo.getUnits()); + e.setType(ProtoUtils.converToProtoFormat(resInfo.getResourceType())); + e.setValue(resInfo.getValue()); builder.addResourceValueMap(e); } } @@ -259,4 +264,14 @@ private void mergeLocalToProto() { proto = builder.build(); viaProto = true; } + + /** + * Get the index to access specific resource information faster. + * @param resource Name of the resource + * @return Index + */ + @Override + public Integer getResourceIndex(String resource) { + return ResourceUtils.getResourceTypeIndex().get(resource); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 86cf872..4d1b755 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -42,6 +42,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; /** * Helper class to read the resource-types to be supported by the system. @@ -66,7 +67,9 @@ } private static volatile Object lock; + private static Map indexForResourceInformation = null; private static Map readOnlyResources; + private static ResourceInformation[] readOnlyResourcesArray; private static volatile Object nodeLock; private static Map readOnlyNodeResources; @@ -252,6 +255,32 @@ static void initializeResourcesMap(Configuration conf, setMinimumAllocationForMandatoryResources(resourceInformationMap, conf); setMaximumAllocationForMandatoryResources(resourceInformationMap, conf); readOnlyResources = Collections.unmodifiableMap(resourceInformationMap); + readOnlyResourcesArray = resourceInformationMap.values() + .toArray(new ResourceInformation[readOnlyResources.size()]); + updateResourceTypeIndex(); + } + + private static void updateResourceTypeIndex() { + if (indexForResourceInformation == null) { + indexForResourceInformation = new ConcurrentHashMap(); + } + indexForResourceInformation.clear(); + + int index = 0; + for (ResourceInformation resInfo : readOnlyResourcesArray) { + indexForResourceInformation.put(resInfo.getName(), index); + index++; + } + } + + /** + * Get associate index of resource types such memory, cpu etc. + * This could help to access each resource types in a resource faster. + * @param name Resource Type name + * @return index Index of Resource Type. + */ + public static Map getResourceTypeIndex() { + return Collections.unmodifiableMap(indexForResourceInformation); } /** @@ -264,6 +293,11 @@ static void initializeResourcesMap(Configuration conf, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); } + public static ResourceInformation[] getResourceTypesArray() { + getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); + return readOnlyResourcesArray; + } + private static Map getResourceTypes( Configuration conf) { return getResourceTypes(conf, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index d143e93..f9e38e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -27,8 +27,10 @@ import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.UnitsConversionUtil; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; @InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" }) @@ -96,8 +98,9 @@ public void setVirtualCores(int virtualCores) { } @Override - public Map getResources() { - return Collections.unmodifiableMap(this.resources); + public ResourceInformation[] getResources() { + return (ResourceInformation[]) this.resources.values() + .toArray(new ResourceInformation[this.resources.size()]); } @Override @@ -135,12 +138,12 @@ public void setResourceValue(String resource, Long value) private Map initResourceMap() { Map tmp = new HashMap<>(); - Map types = ResourceUtils.getResourceTypes(); + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); if (types != null) { - for (Map.Entry entry : types.entrySet()) { - tmp.put(entry.getKey(), - ResourceInformation.newInstance(entry.getValue())); - tmp.get(entry.getKey()).setValue(resourceValue); + for (ResourceInformation entry : types) { + tmp.put(entry.getName(), + ResourceInformation.newInstance(entry)); + tmp.get(entry.getName()).setValue(resourceValue); } } // this is a fix for getVirtualCores returning an int @@ -151,6 +154,10 @@ public void setResourceValue(String resource, Long value) return tmp; } + @Override + public Integer getResourceIndex(String resource) { + return null; + } } public static Resource createResource(int memory) { @@ -197,12 +204,11 @@ public static Resource clone(Resource res) { } public static Resource addTo(Resource lhs, Resource rhs) { - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); + ResourceInformation lhsValue = entry; long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); @@ -219,12 +225,11 @@ public static Resource add(Resource lhs, Resource rhs) { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); + ResourceInformation lhsValue = entry; long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); @@ -263,10 +268,9 @@ public static Resource negate(Resource resource) { } public static Resource multiplyTo(Resource lhs, double by) { - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); - ResourceInformation lhsValue = entry.getValue(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); + ResourceInformation lhsValue = entry; lhs.setResourceValue(name, (long) (lhsValue.getValue() * by)); } return lhs; @@ -282,12 +286,11 @@ public static Resource multiply(Resource lhs, double by) { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); + ResourceInformation lhsValue = entry; long convertedRhs = (long) (UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()) * by); @@ -311,10 +314,9 @@ public static Resource multiplyAndNormalizeDown( public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - for (Map.Entry entry : out.getResources() - .entrySet()) { - String name = entry.getKey(); - ResourceInformation lhsValue = entry.getValue(); + for (ResourceInformation entry : out.getResources()) { + String name = entry.getName(); + ResourceInformation lhsValue = entry; out.setResourceValue(name, (long) (lhsValue.getValue() * by)); } return out; @@ -416,12 +418,11 @@ public static Resource max( } public static boolean fitsIn(Resource smaller, Resource bigger) { - for (Map.Entry entry : smaller.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : smaller.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = bigger.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); + ResourceInformation lhsValue = entry; long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); @@ -442,12 +443,11 @@ public static boolean fitsIn(ResourceCalculator rc, Resource cluster, public static Resource componentwiseMin(Resource lhs, Resource rhs) { Resource ret = createResource(0); - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); + ResourceInformation lhsValue = entry; long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); @@ -463,12 +463,11 @@ public static Resource componentwiseMin(Resource lhs, Resource rhs) { public static Resource componentwiseMax(Resource lhs, Resource rhs) { Resource ret = createResource(0); - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); + ResourceInformation lhsValue = entry; long convertedRhs = UnitsConversionUtil .convert(rhsValue.getUnits(), lhsValue.getUnits(), rhsValue.getValue()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index ff18223..c514cb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -176,16 +176,15 @@ public void updateAggregatePreemptedAppResourceUsage( private void updateUsageMap(Resource allocated, long deltaUsedMillis, Map targetMap) { - for (Map.Entry entry : allocated.getResources() - .entrySet()) { + for (ResourceInformation entry : allocated.getResources()) { AtomicLong resourceUsed; - if (!targetMap.containsKey(entry.getKey())) { + if (!targetMap.containsKey(entry.getName())) { resourceUsed = new AtomicLong(0); - targetMap.put(entry.getKey(), resourceUsed); + targetMap.put(entry.getName(), resourceUsed); } - resourceUsed = targetMap.get(entry.getKey()); - resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis) + resourceUsed = targetMap.get(entry.getName()); + resourceUsed.addAndGet((entry.getValue() * deltaUsedMillis) / DateUtils.MILLIS_PER_SECOND); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 811d9dc..b571322 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -984,13 +984,12 @@ private AggregateAppResourceUsage getRunningAggregateAppResourceUsage() { for (RMContainer rmContainer : this.liveContainers.values()) { long usedMillis = currentTimeMillis - rmContainer.getCreationTime(); Resource resource = rmContainer.getContainer().getResource(); - for (Map.Entry entry : resource - .getResources().entrySet()) { + for (ResourceInformation entry : resource.getResources()) { long value = RMServerUtils - .getOrDefault(resourceSecondsMap, entry.getKey(), 0L); - value += entry.getValue().getValue() * usedMillis + .getOrDefault(resourceSecondsMap, entry.getName(), 0L); + value += entry.getValue() * usedMillis / DateUtils.MILLIS_PER_SECOND; - resourceSecondsMap.put(entry.getKey(), value); + resourceSecondsMap.put(entry.getName(), value); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java index 887b854..d0e97c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java @@ -73,7 +73,7 @@ public ResourceInfo getMaxAllocation() { } public String getSchedulerResourceTypes() { - return minAllocResource.getResource().getResources().keySet().toString(); + return minAllocResource.getResource().getResources().toString(); } public int getMaxClusterLevelAppPriority() { -- 2.10.1 (Apple Git-78)