diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java index 95a8031eb29..a1afb426c24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java @@ -29,13 +29,16 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadLocalRandom; /** * All Resources requested by the container. @@ -66,6 +69,16 @@ private final Map resourcesUploadPolicies = new ConcurrentHashMap<>(); + /** + * Add resources to the current resource set. The ordering returned by the + * Map's Iterator is ignored and resources are added in random order. + * Resources added to the resource set are appended onto the list of existing + * resources. The ordering of existing items in the set are not changed. + * + * @param localResourceMap The local resource map to add from + * @return A map containing the added resources groups by visibility + * @throws URISyntaxException If a resource path is malformed + */ public Map> addResources(Map localResourceMap) throws URISyntaxException { @@ -77,11 +90,14 @@ List privateList = new ArrayList<>(); List appList = new ArrayList<>(); - for (Map.Entry rsrc : localResourceMap.entrySet()) { + List> mapEntries = + new ArrayList<>(localResourceMap.entrySet()); + Collections.shuffle(mapEntries, ThreadLocalRandom.current()); + + for (Map.Entry rsrc : mapEntries) { LocalResource resource = rsrc.getValue(); LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue()); - allResources.putIfAbsent(req, new HashSet<>()); - allResources.get(req).add(rsrc.getKey()); + allResources.computeIfAbsent(req, (key) -> new HashSet<>()).add(rsrc.getKey()); storeSharedCacheUploadPolicy(req, resource.getShouldBeUploadedToSharedCache()); switch (resource.getVisibility()) { @@ -95,6 +111,7 @@ appList.add(req); break; default: + LOG.warn("Unknown visibility: {}", resource.getVisibility()); break; } }