diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 1de7b2fd71a..a25c8acd618 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -327,6 +327,8 @@ public int compareTo(Resource other) { otherResources = other.getResources(); long diff = thisResources.size() - otherResources.size(); if (diff == 0) { + // compare memory and vcores first(in that order) to preserve + // existing behaviour if (thisResources.keySet().equals(otherResources.keySet())) { diff = this.getMemorySize() - other.getMemorySize(); if (diff == 0) { @@ -335,6 +337,11 @@ public int compareTo(Resource other) { if (diff == 0) { for (Map.Entry entry : thisResources .entrySet()) { + if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName()) + || entry.getKey() + .equals(ResourceInformation.VCORES.getName())) { + continue; + } diff = entry.getValue().compareTo(otherResources.get(entry.getKey())); if (diff != 0) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index ab18f2db90b..5f326480506 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -242,7 +242,7 @@ synchronized private void mergeLocalToBuilder() { builder.addResourceValueMap(e); } } - builder.setMemory(this.getMemory()); + builder.setMemory(this.getMemorySize()); builder.setVirtualCores(this.getVirtualCores()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 0412c0fac05..3c4413c4e75 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -397,10 +397,25 @@ private Resource multiplyAndNormalize(Resource r, double by, } @Override - public boolean fitsIn(Resource cluster, - Resource smaller, Resource bigger) { - return smaller.getMemorySize() <= bigger.getMemorySize() - && smaller.getVirtualCores() <= bigger.getVirtualCores(); + public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) { + for (String resource : resourceNames) { + try { + ResourceInformation sResourceInformation = + smaller.getResourceInformation(resource); + ResourceInformation bResourceInformation = + bigger.getResourceInformation(resource); + Long sResourceValue = UnitsConversionUtil + .convert(sResourceInformation.getUnits(), + bResourceInformation.getUnits(), + sResourceInformation.getValue()); + if(sResourceValue > bResourceInformation.getValue()) { + return false; + } + } catch (YarnException ye) { + return false; + } + } + return true; } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 4f56fbb52b3..109287acb0d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -178,7 +178,6 @@ static void initializeResourcesMap(Configuration conf, synchronized (ResourceUtils.class) { if (lock == null) { synchronized (ResourceUtils.class) { - lock = new Object(); Map resources = new HashMap<>(); if (conf == null) { conf = new YarnConfiguration(); @@ -187,10 +186,12 @@ static void initializeResourcesMap(Configuration conf, addResourcesFileToConf(resourceFile, conf); LOG.debug("Found " + resourceFile + ", adding to configuration"); initializeResourcesMap(conf, resources); + lock = new Object(); } catch (FileNotFoundException fe) { LOG.info("Unable to find '" + resourceFile + "'. Falling back to memory and vcores as resources", fe); initializeResourcesMap(conf, resources); + lock = new Object(); } } } @@ -268,12 +269,12 @@ private static String getUnits(String resourceValue) { synchronized (ResourceUtils.class) { if (nodeLock == null) { synchronized (ResourceUtils.class) { - nodeLock = new Object(); Map nodeResources = initializeNodeResourceInformation(conf); addManadtoryResources(nodeResources); checkMandatatoryResources(nodeResources); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); + nodeLock = new Object(); } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 836f87e6c86..73776e7dcb2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -134,16 +134,12 @@ public void setResourceValue(String resource, Long value) private Map initResourceMap() { Map tmp = new HashMap<>(); - // Due to backwards compat, the max value for memory and vcores - // needs to be Integer.MAX_VALUE - int max = resourceValue > Integer.MAX_VALUE ? Integer.MAX_VALUE : - resourceValue.intValue(); Map types = ResourceUtils.getResourceTypes(); if (types != null) { for (Map.Entry entry : types.entrySet()) { tmp.put(entry.getKey(), ResourceInformation.newInstance(entry.getValue())); - tmp.get(entry.getKey()).setValue((long) max); + tmp.get(entry.getKey()).setValue(resourceValue); } } return tmp;